Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

updated longformqa notebook

Arnav Singhvi9fca447244dd
diff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynb
index f3ae4326a..7e4a13608 100644
--- a/examples/longformqa/longformqa_assertions.ipynb
+++ b/examples/longformqa/longformqa_assertions.ipynb
@@ -72,6 +72,7 @@
"try: # When on google Colab, let's clone the notebook so we download the cache.\n",
" import google.colab\n",
" repo_path = 'dspy'\n",
+ " \n",
" !git -C $repo_path pull origin || git clone https://github.com/stanfordnlp/dspy $repo_path\n",
"except:\n",
" repo_path = '.'\n",
@@ -87,8 +88,9 @@
"\n",
"import dspy\n",
"from dspy.predict import Retry\n",
- "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",
"from dspy.datasets import HotPotQA\n",
+ "\n",
+ "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",
"from dsp.utils import EM, normalize_text\n",
"from utils import extract_text_by_citation, correct_citation_format, has_citations, citations_check"
]
@@ -569,7 +571,7 @@
"source": [
"Let's evaluate this with assertions. \n",
"\n",
- "**Note** The pipeline here lies in compiling without assertions to give the teleprompter correct bootstrapped examples by the `answer_correctness` metric and then adding **Assertions** post-compilation. This is represented by passing `LongFormQA()` as the teacher and `LongFormQAWithAssertions()` as the student."
+ "**Note** The pipeline here lies in compiling with **Assertions** to give the teleprompter correct bootstrapped examples by the `answer_correctness` metric and then 'teaching' the student with these correct examples. This is represented by passing `LongFormQA()` as the student and `LongFormQAWithAssertions()` as the teacher."
]
},
{
@@ -580,9 +582,35 @@
"source": [
"longformqa = LongFormQA()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
- "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = longformqa, trainset=trainset, valset=devset[:100])\n",
+ "cited_longformqa = teleprompter.compile(student=longformqa, teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",
"evaluate(cited_longformqa)"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Note** This pipeline on the other hand sets both the teacher and student with `LongFormQAWithAssertions()` to ensure the teacher correctly instructs the student with the right bootstrapped examples and the student has the chance to self-correct with **Assertions** for any examples that are still deemed incorrect. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "longformqa = LongFormQA()\n",
+ "teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
+ "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",
+ "evaluate(cited_longformqa)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {