Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

hotpotqa notebook

manishshettym3d014d4c5fc7
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.py
deleted file mode 100644
index 4ec410fa2..000000000
--- a/assert_hotpotqa.py
+++ /dev/null
@@ -1,142 +0,0 @@
-import dspy
-from dsp.utils import deduplicate
-from dspy.datasets import HotPotQA
-from dspy.predict.retry import Retry
-from dspy.teleprompt import BootstrapFewShot
-
-# pipeline configs
-turbo = dspy.OpenAI(model="gpt-3.5-turbo")
-colbertv2_wiki17_abstracts = dspy.ColBERTv2(
- url="http://20.102.90.50:2017/wiki17_abstracts"
-)
-dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
-
-
-# load dataset
-dataset = HotPotQA(
- train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0
-)
-trainset = [x.with_inputs("question") for x in dataset.train]
-devset = [x.with_inputs("question") for x in dataset.dev]
-
-
-# signatures of dspy modules
-class GenerateAnswer(dspy.Signature):
- """Answer questions with short factoid answers."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- answer = dspy.OutputField(desc="often between 1 and 5 words")
-
-
-class GenerateSearchQuery(dspy.Signature):
- """Write a simple search query that will help answer a complex question."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- query = dspy.OutputField()
-
-
-failure_counts = {
- "answer_exact_match": 0,
- "answer_passage_match": 0,
- "max_hop_length_exceeded": 0,
- "hop_query_similarity_exceeded": 0,
- "failed_prog_assertions": 0,
-}
-
-
-def validate_context_and_answer_and_hops(example, pred, trace=None):
- global failure_counts
-
- try:
- if not dspy.evaluate.answer_exact_match(example, pred):
- failure_counts["answer_exact_match"] += 1
- return False
-
- if not dspy.evaluate.answer_passage_match(example, pred):
- failure_counts["answer_passage_match"] += 1
- return False
-
- hops = [example.question] + [
- outputs.query for *_, outputs in trace if "query" in outputs
- ]
-
- if max([len(h) for h in hops]) > 100:
- failure_counts["max_hop_length_exceeded"] += 1
- return False
-
- if any(
- dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)
- for idx in range(2, len(hops))
- ):
- failure_counts["hop_query_similarity_exceeded"] += 1
- return False
- except:
- failure_counts["failed_prog_assertions"] += 1
- print("failed prog assertions:", example.question)
- return False
-
- return True
-
-
-def validate_query_distinction_local(previous_queries, query):
- """check if query is distinct from previous queries"""
- if previous_queries == []:
- return True
- if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):
- return False
- return True
-
-
-# declaration of dspy program
-class SimplifiedBaleen(dspy.Module):
-
- def __init__(self, passages_per_hop=2, max_hops=2):
- super().__init__()
-
- self.generate_query = [
- dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)
- ]
- self.retrieve = dspy.Retrieve(k=passages_per_hop)
- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
- self.max_hops = max_hops
-
- def forward(self, question):
- context = []
- prev_queries = [question]
-
- for hop in range(self.max_hops):
- query = self.generate_query[hop](context=context, question=question).query
-
- dspy.Suggest(
- len(query) <= 100,
- "Query should be short and less than 100 characters",
- )
-
- dspy.Suggest(
- validate_query_distinction_local(prev_queries, query),
- "Query should be distinct from: "
- + "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),
- )
-
- prev_queries.append(query)
- passages = self.retrieve(query).passages
- context = deduplicate(context + passages)
-
- pred = self.generate_answer(context=context, question=question)
-
- return dspy.Prediction(context=context, answer=pred.answer)
-
-
-teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
-
-compiled_baleen = teleprompter.compile(
- student=SimplifiedBaleen().map_named_predictors(Retry),
- teacher=SimplifiedBaleen(passages_per_hop=2).map_named_predictors(Retry),
- trainset=trainset,
-)
-
-print("=" * 50, "Validation Failures", "=" * 50)
-for failure_type, count in failure_counts.items():
- print(f"{failure_type}: {count}")
diff --git a/hotpotqa_with_assertions.ipynb b/hotpotqa_with_assertions.ipynb
new file mode 100644
index 000000000..0d35b4da5
--- /dev/null
+++ b/hotpotqa_with_assertions.ipynb
@@ -0,0 +1,300 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dspy\n",
+ "from dsp.utils import deduplicate\n",
+ "from dspy.datasets import HotPotQA\n",
+ "from dspy.predict.retry import Retry\n",
+ "from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",
+ "from dspy.evaluate.evaluate import Evaluate\n",
+ "\n",
+ "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import openai\n",
+ "openai.api_key = os.getenv('OPENAI_API_KEY')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
+ "dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
+ "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",
+ "dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",
+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",
+ "devset = [x.with_inputs('question') for x in dataset.dev]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Suggestion helper functions and Teleprompter metric\n",
+ "\n",
+ "def validate_query_distinction_local(previous_queries, query):\n",
+ " \"\"\"check if query is distinct from previous queries\"\"\"\n",
+ " if previous_queries == []:\n",
+ " return True\n",
+ " if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):\n",
+ " return False\n",
+ " return True\n",
+ "\n",
+ "\n",
+ "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",
+ " if not dspy.evaluate.answer_exact_match(example, pred):\n",
+ " return False\n",
+ "\n",
+ " if not dspy.evaluate.answer_passage_match(example, pred):\n",
+ " return False\n",
+ "\n",
+ " return True\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extrinsic metrics\n",
+ "\n",
+ "def gold_passages_retrieved(example, pred, trace=None):\n",
+ " gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",
+ " found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",
+ "\n",
+ " return gold_titles.issubset(found_titles)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# signatures of dspy modules\n",
+ "class GenerateAnswer(dspy.Signature):\n",
+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",
+ "\n",
+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
+ " question = dspy.InputField()\n",
+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
+ "\n",
+ "\n",
+ "class GenerateSearchQuery(dspy.Signature):\n",
+ " \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",
+ "\n",
+ " context = dspy.InputField(desc=\"may contain relevant facts\")\n",
+ " question = dspy.InputField()\n",
+ " query = dspy.OutputField()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SimplifiedBaleen(dspy.Module):\n",
+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",
+ " super().__init__()\n",
+ "\n",
+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
+ " self.max_hops = max_hops\n",
+ "\n",
+ " def forward(self, question):\n",
+ " context = []\n",
+ " for hop in range(self.max_hops):\n",
+ " query = self.generate_query[hop](context=context, question=question).query\n",
+ " passages = self.retrieve(query).passages\n",
+ " context = deduplicate(context + passages)\n",
+ " pred = self.generate_answer(context=context, question=question)\n",
+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
+ " return pred"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SimplifiedBaleenAssertions(dspy.Module):\n",
+ " def __init__(self, passages_per_hop=2, max_hops=2):\n",
+ " super().__init__()\n",
+ " self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
+ " self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
+ " self.max_hops = max_hops\n",
+ "\n",
+ " def forward(self, question):\n",
+ " context = []\n",
+ " prev_queries = [question]\n",
+ "\n",
+ " for hop in range(self.max_hops):\n",
+ " query = self.generate_query[hop](context=context, question=question).query\n",
+ "\n",
+ " dspy.Suggest(\n",
+ " len(query) <= 100,\n",
+ " \"Query should be short and less than 100 characters\",\n",
+ " )\n",
+ "\n",
+ " dspy.Suggest(\n",
+ " validate_query_distinction_local(prev_queries, query),\n",
+ " \"Query should be distinct from: \"\n",
+ " + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",
+ " )\n",
+ "\n",
+ " prev_queries.append(query)\n",
+ " passages = self.retrieve(query).passages\n",
+ " context = deduplicate(context + passages)\n",
+ "\n",
+ " pred = self.generate_answer(context=context, question=question)\n",
+ " pred = dspy.Prediction(context=context, answer=pred.answer)\n",
+ " return pred\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate(module):\n",
+ " retrieval_score = evaluate_on_hotpotqa(\n",
+ " module, metric=gold_passages_retrieved\n",
+ " )\n",
+ " \n",
+ " accuracy_score = evaluate_on_hotpotqa(\n",
+ " module, metric=dspy.evaluate.answer_exact_match\n",
+ " )\n",
+ "\n",
+ " print(f\"## Retrieval Score: {retrieval_score}\")\n",
+ " print(f\"## Accuracy Score: {accuracy_score}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# No Compilation + No Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "evaluate(baleen)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# No Compilation + Yes Assertion\n",
+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
+ "evaluate(baleen_with_assertions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Yes Compilation + No Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ " metric=validate_context_and_answer_and_hops,\n",
+ " max_bootstrapped_demos=2,\n",
+ " num_candidate_programs=6,\n",
+ ")\n",
+ "\n",
+ "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
+ "evaluate(compiled_baleen)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Yes Compilation + Yes Assertion\n",
+ "baleen = SimplifiedBaleen()\n",
+ "teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ " metric=validate_context_and_answer_and_hops,\n",
+ " max_bootstrapped_demos=2,\n",
+ " num_candidate_programs=6,\n",
+ ")\n",
+ "compiled_baleen = teleprompter.compile(\n",
+ " student=assert_transform_module(\n",
+ " SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
+ " suggest_backtrack_handler,\n",
+ " ),\n",
+ " teacher=baleen,\n",
+ " trainset=trainset,\n",
+ " valset=devset[:100]\n",
+ ")\n",
+ "evaluate(compiled_baleen)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}