Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Update HotPotQA with more demos

Shangyintd295e61bbf52
diff --git a/hotpotqa_with_assertions.ipynb b/hotpotqa_with_assertions.ipynb
index 5c600a4c5..faa8ed8a6 100644
--- a/hotpotqa_with_assertions.ipynb
+++ b/hotpotqa_with_assertions.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
@@ -18,7 +18,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@@ -29,7 +29,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@@ -41,7 +41,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -52,7 +52,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -79,7 +79,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
@@ -94,7 +94,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -132,7 +132,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -168,7 +168,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@@ -215,7 +215,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@@ -224,7 +224,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
@@ -279,11 +279,11 @@
"baleen = SimplifiedBaleen()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(\n",
" metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
+ " max_bootstrapped_demos=4,\n",
" num_candidate_programs=6,\n",
")\n",
"\n",
- "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
+ "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n",
"evaluate(compiled_baleen)"
]
},
@@ -294,10 +294,11 @@
"outputs": [],
"source": [
"# Yes Compilation + Yes Assertion\n",
+ "from functools import partial\n",
"baleen = SimplifiedBaleen()\n",
- "teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ "teleprompter = BootstrapFewShot(\n",
" metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
+ " max_bootstrapped_demos=4,\n",
" num_candidate_programs=6,\n",
")\n",
"compiled_baleen = teleprompter.compile(\n",
@@ -305,9 +306,13 @@
" SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
" suggest_backtrack_handler,\n",
" ),\n",
+ " # teacher=assert_transform_module(\n",
+ " # SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
+ " # partial(suggest_backtrack_handler, bypass_suggest=False)\n",
+ " # ),\n",
" teacher=baleen,\n",
" trainset=trainset,\n",
- " valset=devset[:100]\n",
+ " valset=devset\n",
")\n",
"evaluate(compiled_baleen)\n"
]