DSPy: The framework for programming—not prompting—language models
Python
hotpotqa notebook
manishshettym3d014d4c5fc7
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.pydeleted file mode 100644index 4ec410fa2..000000000--- a/assert_hotpotqa.py+++ /dev/null@@ -1,142 +0,0 @@-import dspy-from dsp.utils import deduplicate-from dspy.datasets import HotPotQA-from dspy.predict.retry import Retry-from dspy.teleprompt import BootstrapFewShot--# pipeline configs-turbo = dspy.OpenAI(model="gpt-3.5-turbo")-colbertv2_wiki17_abstracts = dspy.ColBERTv2(- url="http://20.102.90.50:2017/wiki17_abstracts"-)-dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)---# load dataset-dataset = HotPotQA(- train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0-)-trainset = [x.with_inputs("question") for x in dataset.train]-devset = [x.with_inputs("question") for x in dataset.dev]---# signatures of dspy modules-class GenerateAnswer(dspy.Signature):- """Answer questions with short factoid answers."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- answer = dspy.OutputField(desc="often between 1 and 5 words")---class GenerateSearchQuery(dspy.Signature):- """Write a simple search query that will help answer a complex question."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- query = dspy.OutputField()---failure_counts = {- "answer_exact_match": 0,- "answer_passage_match": 0,- "max_hop_length_exceeded": 0,- "hop_query_similarity_exceeded": 0,- "failed_prog_assertions": 0,-}---def validate_context_and_answer_and_hops(example, pred, trace=None):- global failure_counts-- try:- if not dspy.evaluate.answer_exact_match(example, pred):- failure_counts["answer_exact_match"] += 1- return False-- if not dspy.evaluate.answer_passage_match(example, pred):- failure_counts["answer_passage_match"] += 1- return False-- hops = [example.question] + [- outputs.query for *_, outputs in trace if "query" in outputs- ]-- if max([len(h) for h in hops]) > 100:- failure_counts["max_hop_length_exceeded"] += 1- return False-- if any(- dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)- for idx in range(2, len(hops))- ):- failure_counts["hop_query_similarity_exceeded"] += 1- return False- except:- failure_counts["failed_prog_assertions"] += 1- print("failed prog assertions:", example.question)- return False-- return True---def validate_query_distinction_local(previous_queries, query):- """check if query is distinct from previous queries"""- if previous_queries == []:- return True- if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):- return False- return True---# declaration of dspy program-class SimplifiedBaleen(dspy.Module):-- def __init__(self, passages_per_hop=2, max_hops=2):- super().__init__()-- self.generate_query = [- dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)- ]- self.retrieve = dspy.Retrieve(k=passages_per_hop)- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)- self.max_hops = max_hops-- def forward(self, question):- context = []- prev_queries = [question]-- for hop in range(self.max_hops):- query = self.generate_query[hop](context=context, question=question).query-- dspy.Suggest(- len(query) <= 100,- "Query should be short and less than 100 characters",- )-- dspy.Suggest(- validate_query_distinction_local(prev_queries, query),- "Query should be distinct from: "- + "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),- )-- prev_queries.append(query)- passages = self.retrieve(query).passages- context = deduplicate(context + passages)-- pred = self.generate_answer(context=context, question=question)-- return dspy.Prediction(context=context, answer=pred.answer)---teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)--compiled_baleen = teleprompter.compile(- student=SimplifiedBaleen().map_named_predictors(Retry),- teacher=SimplifiedBaleen(passages_per_hop=2).map_named_predictors(Retry),- trainset=trainset,-)--print("=" * 50, "Validation Failures", "=" * 50)-for failure_type, count in failure_counts.items():- print(f"{failure_type}: {count}")diff --git a/hotpotqa_with_assertions.ipynb b/hotpotqa_with_assertions.ipynbnew file mode 100644index 000000000..0d35b4da5--- /dev/null+++ b/hotpotqa_with_assertions.ipynb@@ -0,0 +1,300 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "import dspy\n",+ "from dsp.utils import deduplicate\n",+ "from dspy.datasets import HotPotQA\n",+ "from dspy.predict.retry import Retry\n",+ "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",+ "from dspy.evaluate.evaluate import Evaluate\n",+ "\n",+ "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "import os\n",+ "import openai\n",+ "openai.api_key = os.getenv('OPENAI_API_KEY')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",+ "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",+ "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",+ "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",+ "devset = [x.with_inputs('question') for x in dataset.dev]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Suggestion helper functions and Teleprompter metric\n",+ "\n",+ "def validate_query_distinction_local(previous_queries, query):\n",+ " \"\"\"check if query is distinct from previous queries\"\"\"\n",+ " if previous_queries == []:\n",+ " return True\n",+ " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",+ " return False\n",+ " return True\n",+ "\n",+ "\n",+ "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",+ " if not dspy.evaluate.answer_exact_match(example, pred):\n",+ " return False\n",+ "\n",+ " if not dspy.evaluate.answer_passage_match(example, pred):\n",+ " return False\n",+ "\n",+ " return True\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Extrinsic metrics\n",+ "\n",+ "def gold_passages_retrieved(example, pred, trace=None):\n",+ " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",+ " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",+ "\n",+ " return gold_titles.issubset(found_titles)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# signatures of dspy modules\n",+ "class GenerateAnswer(dspy.Signature):\n",+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",+ "\n",+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",+ " question = dspy.InputField()\n",+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",+ "\n",+ "\n",+ "class GenerateSearchQuery(dspy.Signature):\n",+ " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",+ "\n",+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",+ " question = dspy.InputField()\n",+ " query = dspy.OutputField()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "class SimplifiedBaleen(dspy.Module):\n",+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",+ " super().__init__()\n",+ "\n",+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",+ " self.max_hops = max_hops\n",+ "\n",+ " def forward(self, question):\n",+ " context = []\n",+ " for hop in range(self.max_hops):\n",+ " query = self.generate_query[hop](context=context, question=question).query\n",+ " passages = self.retrieve(query).passages\n",+ " context = deduplicate(context + passages)\n",+ " pred = self.generate_answer(context=context, question=question)\n",+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",+ " return pred"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "class SimplifiedBaleenAssertions(dspy.Module):\n",+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",+ " super().__init__()\n",+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",+ " self.max_hops = max_hops\n",+ "\n",+ " def forward(self, question):\n",+ " context = []\n",+ " prev_queries = [question]\n",+ "\n",+ " for hop in range(self.max_hops):\n",+ " query = self.generate_query[hop](context=context, question=question).query\n",+ "\n",+ " dspy.Suggest(\n",+ " len(query) <= 100,\n",+ " \"Query should be short and less than 100 characters\",\n",+ " )\n",+ "\n",+ " dspy.Suggest(\n",+ " validate_query_distinction_local(prev_queries, query),\n",+ " \"Query should be distinct from: \"\n",+ " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",+ " )\n",+ "\n",+ " prev_queries.append(query)\n",+ " passages = self.retrieve(query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " pred = self.generate_answer(context=context, question=question)\n",+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",+ " return pred\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "def evaluate(module):\n",+ " retrieval_score = evaluate_on_hotpotqa(\n",+ " module, metric=gold_passages_retrieved\n",+ " )\n",+ " \n",+ " accuracy_score = evaluate_on_hotpotqa(\n",+ " module, metric=dspy.evaluate.answer_exact_match\n",+ " )\n",+ "\n",+ " print(f\"## Retrieval Score: {retrieval_score}\")\n",+ " print(f\"## Accuracy Score: {accuracy_score}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# No Compilation + No Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "evaluate(baleen)\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# No Compilation + Yes Assertion\n",+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",+ "evaluate(baleen_with_assertions)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Yes Compilation + No Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",+ " metric=validate_context_and_answer_and_hops,\n",+ " max_bootstrapped_demos=2,\n",+ " num_candidate_programs=6,\n",+ ")\n",+ "\n",+ "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",+ "evaluate(compiled_baleen)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Yes Compilation + Yes Assertion\n",+ "baleen = SimplifiedBaleen()\n",+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",+ " metric=validate_context_and_answer_and_hops,\n",+ " max_bootstrapped_demos=2,\n",+ " num_candidate_programs=6,\n",+ ")\n",+ "compiled_baleen = teleprompter.compile(\n",+ " student=assert_transform_module(\n",+ " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",+ " suggest_backtrack_handler,\n",+ " ),\n",+ " teacher=baleen,\n",+ " trainset=trainset,\n",+ " valset=devset[:100]\n",+ ")\n",+ "evaluate(compiled_baleen)\n"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "Python 3",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.5"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}