Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Revert back changes to intrinsic metrics

Shangyint7641d37eda15
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
index 36c3daa20..9e07b11bb 100644
--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
@@ -52,7 +52,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -79,7 +79,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -94,7 +94,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +117,22 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def all_queries_distinct(prev_queries):\n",
+ " query_distinct = True\n",
+ " for i, query in enumerate(prev_queries):\n",
+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",
+ " query_distinct = False\n",
+ " break\n",
+ " return query_distinct"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -130,12 +145,22 @@
" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
" self.max_hops = max_hops\n",
"\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
" def forward(self, question):\n",
" context = []\n",
+ " prev_queries = [question]\n",
+ "\n",
" for hop in range(self.max_hops):\n",
" query = self.generate_query[hop](context=context, question=question).query\n",
+ " prev_queries.append(query)\n",
" passages = self.retrieve(query).passages\n",
" context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
+ " \n",
" pred = self.generate_answer(context=context, question=question)\n",
" pred = dspy.Prediction(context=context, answer=pred.answer)\n",
" return pred"
@@ -143,7 +168,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
@@ -155,6 +180,9 @@
" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
" self.max_hops = max_hops\n",
"\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
" def forward(self, question):\n",
" context = []\n",
" prev_queries = [question]\n",
@@ -176,6 +204,9 @@
" prev_queries.append(query)\n",
" passages = self.retrieve(query).passages\n",
" context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
"\n",
" pred = self.generate_answer(context=context, question=question)\n",
" pred = dspy.Prediction(context=context, answer=pred.answer)\n",
@@ -184,28 +215,34 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"
+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def evaluate(module):\n",
+ " module.passed_suggestions = 0\n",
+ "\n",
" retrieval_score = evaluate_on_hotpotqa(\n",
" module, metric=gold_passages_retrieved\n",
" )\n",
" \n",
+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",
+ "\n",
" accuracy_score = evaluate_on_hotpotqa(\n",
" module, metric=dspy.evaluate.answer_exact_match\n",
" )\n",
"\n",
+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",
+ "\n",
" print(f\"## Retrieval Score: {retrieval_score}\")\n",
" print(f\"## Accuracy Score: {accuracy_score}\")"
]
@@ -232,6 +269,15 @@
"evaluate(baleen_with_assertions)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "max_bootstrapped_demos = 2"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -242,11 +288,11 @@
"baleen = SimplifiedBaleen()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(\n",
" metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",
" num_candidate_programs=6,\n",
")\n",
"\n",
- "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
+ "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n",
"evaluate(compiled_baleen)"
]
},
@@ -260,7 +306,7 @@
"baleen = SimplifiedBaleen()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(\n",
" metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
+ " max_bootstrapped_demos=max_bootstrapped_demos,\n",
" num_candidate_programs=6,\n",
")\n",
"compiled_baleen = teleprompter.compile(\n",
@@ -270,9 +316,9 @@
" ),\n",
" teacher=baleen,\n",
" trainset=trainset,\n",
- " valset=devset[:100]\n",
+ " valset=devset\n",
")\n",
- "evaluate(compiled_baleen)"
+ "evaluate(compiled_baleen)\n"
]
}
],
@@ -292,7 +338,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "version": "3.11.5"
}
},
"nbformat": 4,