Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Revert back changes to intrinsic metrics

Shangyintdad446f8f264
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
index 845b4e240..350647a51 100644
--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
@@ -1,655 +1,346 @@
{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "%load_ext autoreload\n",
- "%autoreload 2\n",
- "\n",
- "import dspy\n",
- "from dsp.utils import deduplicate\n",
- "from dspy.datasets import HotPotQA\n",
- "from dspy.predict.retry import Retry\n",
- "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",
- "from dspy.evaluate.evaluate import Evaluate\n",
- "\n",
- "from dspy.primitives.assertions import assert_transform_module, backtrack_handler"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import openai\n",
- "openai.api_key = os.getenv('OPENAI_API_KEY')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
- "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
- "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",
- "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",
- "trainset = [x.with_inputs('question') for x in dataset.train]\n",
- "devset = [x.with_inputs('question') for x in dataset.dev]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Suggestion helper functions and Teleprompter metric\n",
- "\n",
- "def validate_query_distinction_local(previous_queries, query):\n",
- " \"\"\"check if query is distinct from previous queries\"\"\"\n",
- " if previous_queries == []:\n",
- " return True\n",
- " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",
- " return False\n",
- " return True\n",
- "\n",
- "\n",
- "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",
- " if not dspy.evaluate.answer_exact_match(example, pred):\n",
- " return False\n",
- "\n",
- " if not dspy.evaluate.answer_passage_match(example, pred):\n",
- " return False\n",
- "\n",
- " return True\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Extrinsic metrics\n",
- "\n",
- "def gold_passages_retrieved(example, pred, trace=None):\n",
- " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",
- " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",
- "\n",
- " return gold_titles.issubset(found_titles)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [],
- "source": [
- "# signatures of dspy modules\n",
- "class GenerateAnswer(dspy.Signature):\n",
- " \"\"\"Answer questions with short factoid answers.\"\"\"\n",
- "\n",
- " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
- " question = dspy.InputField()\n",
- " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
- "\n",
- "\n",
- "class GenerateSearchQuery(dspy.Signature):\n",
- " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",
- "\n",
- " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
- " question = dspy.InputField()\n",
- " query = dspy.OutputField()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [],
- "source": [
- "class SimplifiedBaleen(dspy.Module):\n",
- " def __init__(self, passages_per_hop=2, max_hops=2):\n",
- " super().__init__()\n",
- "\n",
- " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
- " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
- " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
- " self.max_hops = max_hops\n",
- "\n",
- " def forward(self, question):\n",
- " context = []\n",
- " for hop in range(self.max_hops):\n",
- " query = self.generate_query[hop](context=context, question=question).query\n",
- " passages = self.retrieve(query).passages\n",
- " context = deduplicate(context + passages)\n",
- " pred = self.generate_answer(context=context, question=question)\n",
- " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
- " return pred"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {},
- "outputs": [],
- "source": [
- "class SimplifiedBaleenAssertions(dspy.Module):\n",
- " def __init__(self, passages_per_hop=2, max_hops=2):\n",
- " super().__init__()\n",
- " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
- " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
- " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
- " self.max_hops = max_hops\n",
- "\n",
- " def forward(self, question):\n",
- " context = []\n",
- " prev_queries = [question]\n",
- "\n",
- " for hop in range(self.max_hops):\n",
- " query = self.generate_query[hop](context=context, question=question).query\n",
- "\n",
- " dspy.Suggest(\n",
- " len(query) <= 100,\n",
- " \"Query should be short and less than 100 characters\",\n",
- " )\n",
- "\n",
- " dspy.Suggest(\n",
- " validate_query_distinction_local(prev_queries, query),\n",
- " \"Query should be distinct from: \"\n",
- " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",
- " is_metric=True,\n",
- " )\n",
- "\n",
- " prev_queries.append(query)\n",
- " passages = self.retrieve(query).passages\n",
- " context = deduplicate(context + passages)\n",
- "\n",
- " pred = self.generate_answer(context=context, question=question)\n",
- " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
- " return pred\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [],
- "source": [
- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [],
- "source": [
- "def evaluate(module):\n",
- " retrieval_score = evaluate_on_hotpotqa(\n",
- " module, metric=gold_passages_retrieved\n",
- " )\n",
- " \n",
- " accuracy_score = evaluate_on_hotpotqa(\n",
- " module, metric=dspy.evaluate.answer_exact_match\n",
- " )\n",
- "\n",
- " print(f\"## Retrieval Score: {retrieval_score}\")\n",
- " print(f\"## Accuracy Score: {accuracy_score}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # No Compilation + No Assertion\n",
- "# baleen = SimplifiedBaleen()\n",
- "# evaluate(baleen)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # No Compilation + Yes Assertion\n",
- "# baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
- "# evaluate(baleen_with_assertions)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Yes Compilation + No Assertion\n",
- "# baleen = SimplifiedBaleen()\n",
- "# teleprompter = BootstrapFewShotWithRandomSearch(\n",
- "# metric=validate_context_and_answer_and_hops,\n",
- "# max_bootstrapped_demos=2,\n",
- "# num_candidate_programs=6,\n",
- "# )\n",
- "\n",
- "# compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
- "# evaluate(compiled_baleen)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 21,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Going to sample between 1 and 2 traces per predictor.\n",
- "Will attempt to train 6 candidate sets.\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 524.07it/s]\n",
- "/Users/manishs/Projects/dspy/dspy/evaluate/evaluate.py:130: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
- " df = df.applymap(truncate_cell)\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 43 / 100 (43.0%)\n",
- "Score: 43.0 for set: [0, 0, 0]\n",
- "New best score: 43.0 for seed -3\n",
- "Scores so far: [43.0]\n",
- "Best score: 43.0\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:00<00:00, 514.39it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 42 / 100 (42.0%)\n",
- "Score: 42.0 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0]\n",
- "Best score: 43.0\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%|▏ | 4/300 [00:00<00:00, 1176.85it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 2 full traces after 5 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 4\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:00<00:00, 517.62it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 45 / 100 (45.0%)\n",
- "Score: 44.2 for set: [16, 16, 16]\n",
- "New best score: 44.2 for seed -1\n",
- "Scores so far: [43.0, 42.0, 44.2]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.51\n",
- "Average of max per entry across top 3 scores: 0.54\n",
- "Average of max per entry across top 5 scores: 0.54\n",
- "Average of max per entry across top 8 scores: 0.54\n",
- "Average of max per entry across top 9999 scores: 0.54\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%| | 3/300 [00:00<00:00, 1135.03it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 2 full traces after 4 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 4\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:00<00:00, 486.05it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 44 / 100 (44.0%)\n",
- "Score: 43.2 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.56\n",
- "Average of max per entry across top 8 scores: 0.56\n",
- "Average of max per entry across top 9999 scores: 0.56\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%| | 2/300 [00:00<00:00, 1088.72it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 1 full traces after 3 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 0\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 468.94it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 40 / 100 (40.0%)\n",
- "Score: 40.0 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.58\n",
- "Average of max per entry across top 8 scores: 0.58\n",
- "Average of max per entry across top 9999 scores: 0.58\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%| | 2/300 [00:00<00:00, 1038.84it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 1 full traces after 3 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 2\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:00<00:00, 537.95it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 38 / 100 (38.0%)\n",
- "Score: 37.6 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.58\n",
- "Average of max per entry across top 8 scores: 0.58\n",
- "Average of max per entry across top 9999 scores: 0.58\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%|▏ | 4/300 [00:00<00:00, 1060.71it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 1 full traces after 5 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 1\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:00<00:00, 428.19it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 37 / 100 (37.0%)\n",
- "Score: 36.8 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.58\n",
- "Average of max per entry across top 8 scores: 0.59\n",
- "Average of max per entry across top 9999 scores: 0.59\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%| | 3/300 [00:00<00:00, 1050.50it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 1 full traces after 4 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 1\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 555.81it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 43 / 100 (43.0%)\n",
- "Score: 42.8 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.57\n",
- "Average of max per entry across top 8 scores: 0.6\n",
- "Average of max per entry across top 9999 scores: 0.6\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 1%| | 2/300 [00:00<00:00, 616.54it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 2 full traces after 3 examples in round 0.\n",
- "# of suggestion failures during bootstrapping: 2\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 460.90it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 40 / 100 (40.0%)\n",
- "Score: 39.6 for set: [16, 16, 16]\n",
- "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8, 39.6]\n",
- "Best score: 44.2\n",
- "Average of max per entry across top 1 scores: 0.45\n",
- "Average of max per entry across top 2 scores: 0.52\n",
- "Average of max per entry across top 3 scores: 0.55\n",
- "Average of max per entry across top 5 scores: 0.57\n",
- "Average of max per entry across top 8 scores: 0.6\n",
- "Average of max per entry across top 9999 scores: 0.61\n",
- "9 candidate programs found.\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 126 / 300 (42.0): 100%|██████████| 300/300 [00:00<00:00, 595.84it/s]\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 126 / 300 (42.0%)\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Average Metric: 150 / 300 (50.0): 100%|██████████| 300/300 [00:00<00:00, 618.44it/s]"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Average Metric: 150 / 300 (50.0%)\n",
- "## Retrieval Score: 42.0\n",
- "## Accuracy Score: 50.0\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n"
- ]
- }
- ],
- "source": [
- "# Yes Compilation + Yes Assertion\n",
- "baleen = SimplifiedBaleen()\n",
- "teleprompter = BootstrapFewShotWithRandomSearch(\n",
- " metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
- " num_candidate_programs=6,\n",
- ")\n",
- "compiled_baleen = teleprompter.compile(\n",
- " student=assert_transform_module(\n",
- " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
- " backtrack_handler,\n",
- " ),\n",
- " teacher=baleen,\n",
- " trainset=trainset,\n",
- " valset=devset[:100]\n",
- ")\n",
- "evaluate(compiled_baleen)"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dspy\n",
+ "from dsp.utils import deduplicate\n",
+ "from dspy.datasets import HotPotQA\n",
+ "from dspy.predict.retry import Retry\n",
+ "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",
+ "from dspy.evaluate.evaluate import Evaluate\n",
+ "\n",
+ "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import openai\n",
+ "openai.api_key = os.getenv('OPENAI_API_KEY')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
+ "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
+ "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",
+ "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",
+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",
+ "devset = [x.with_inputs('question') for x in dataset.dev]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Suggestion helper functions and Teleprompter metric\n",
+ "\n",
+ "def validate_query_distinction_local(previous_queries, query):\n",
+ " \"\"\"check if query is distinct from previous queries\"\"\"\n",
+ " if previous_queries == []:\n",
+ " return True\n",
+ " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",
+ " return False\n",
+ " return True\n",
+ "\n",
+ "\n",
+ "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",
+ " if not dspy.evaluate.answer_exact_match(example, pred):\n",
+ " return False\n",
+ "\n",
+ " if not dspy.evaluate.answer_passage_match(example, pred):\n",
+ " return False\n",
+ "\n",
+ " return True\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extrinsic metrics\n",
+ "\n",
+ "def gold_passages_retrieved(example, pred, trace=None):\n",
+ " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",
+ " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",
+ "\n",
+ " return gold_titles.issubset(found_titles)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# signatures of dspy modules\n",
+ "class GenerateAnswer(dspy.Signature):\n",
+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",
+ "\n",
+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
+ " question = dspy.InputField()\n",
+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
+ "\n",
+ "\n",
+ "class GenerateSearchQuery(dspy.Signature):\n",
+ " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",
+ "\n",
+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
+ " question = dspy.InputField()\n",
+ " query = dspy.OutputField()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def all_queries_distinct(prev_queries):\n",
+ " query_distinct = True\n",
+ " for i, query in enumerate(prev_queries):\n",
+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",
+ " query_distinct = False\n",
+ " break\n",
+ " return query_distinct"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SimplifiedBaleen(dspy.Module):\n",
+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",
+ " super().__init__()\n",
+ "\n",
+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
+ " self.max_hops = max_hops\n",
+ "\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
+ " def forward(self, question):\n",
+ " context = []\n",
+ " prev_queries = [question]\n",
+ "\n",
+ " for hop in range(self.max_hops):\n",
+ " query = self.generate_query[hop](context=context, question=question).query\n",
+ " prev_queries.append(query)\n",
+ " passages = self.retrieve(query).passages\n",
+ " context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
+ " \n",
+ " pred = self.generate_answer(context=context, question=question)\n",
+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
+ " return pred"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SimplifiedBaleenAssertions(dspy.Module):\n",
+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",
+ " super().__init__()\n",
+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
+ " self.max_hops = max_hops\n",
+ "\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
+ " def forward(self, question):\n",
+ " context = []\n",
+ " prev_queries = [question]\n",
+ "\n",
+ " for hop in range(self.max_hops):\n",
+ " query = self.generate_query[hop](context=context, question=question).query\n",
+ "\n",
+ " dspy.Suggest(\n",
+ " len(query) <= 100,\n",
+ " \"Query should be short and less than 100 characters\",\n",
+ " )\n",
+ "\n",
+ " dspy.Suggest(\n",
+ " validate_query_distinction_local(prev_queries, query),\n",
+ " \"Query should be distinct from: \"\n",
+ " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",
+ " )\n",
+ "\n",
+ " prev_queries.append(query)\n",
+ " passages = self.retrieve(query).passages\n",
+ " context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
+ "\n",
+ " pred = self.generate_answer(context=context, question=question)\n",
+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
+ " return pred\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate(module):\n",
+ " module.passed_suggestions = 0\n",
+ "\n",
+ " retrieval_score = evaluate_on_hotpotqa(\n",
+ " module, metric=gold_passages_retrieved\n",
+ " )\n",
+ " \n",
+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",
+ "\n",
+ " accuracy_score = evaluate_on_hotpotqa(\n",
+ " module, metric=dspy.evaluate.answer_exact_match\n",
+ " )\n",
+ "\n",
+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",
+ "\n",
+ " print(f\"## Retrieval Score: {retrieval_score}\")\n",
+ " print(f\"## Accuracy Score: {accuracy_score}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# No Compilation + No Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "evaluate(baleen)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# No Compilation + Yes Assertion\n",
+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
+ "evaluate(baleen_with_assertions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "max_bootstrapped_demos = 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Yes Compilation + No Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ " metric=validate_context_and_answer_and_hops,\n",
+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",
+ " num_candidate_programs=6,\n",
+ ")\n",
+ "\n",
+ "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n",
+ "evaluate(compiled_baleen)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Yes Compilation + Yes Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ " metric=validate_context_and_answer_and_hops,\n",
+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",
+ " num_candidate_programs=6,\n",
+ ")\n",
+ "compiled_baleen = teleprompter.compile(\n",
+ " student=assert_transform_module(\n",
+ " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
+ " suggest_backtrack_handler,\n",
+ " ),\n",
+ " teacher=baleen,\n",
+ " trainset=trainset,\n",
+ " valset=devset\n",
+ ")\n",
+ "evaluate(compiled_baleen)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+ }
\ No newline at end of file