Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

update hotpotqa intrinsic metrics

Shangyint70db7ea2b0e6
diff --git a/hotpotqa_with_assertions.ipynb b/hotpotqa_with_assertions.ipynb
index 0d35b4da5..5c600a4c5 100644
--- a/hotpotqa_with_assertions.ipynb
+++ b/hotpotqa_with_assertions.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -18,7 +18,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -29,7 +29,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -41,7 +41,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -52,7 +52,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -79,7 +79,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -94,7 +94,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +117,22 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def all_queries_distinct(prev_queries):\n",
+ " query_distinct = True\n",
+ " for i, query in enumerate(prev_queries):\n",
+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",
+ " query_distinct = False\n",
+ " break\n",
+ " return query_distinct"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -130,12 +145,22 @@
" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
" self.max_hops = max_hops\n",
"\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
" def forward(self, question):\n",
" context = []\n",
+ " prev_queries = [question]\n",
+ "\n",
" for hop in range(self.max_hops):\n",
" query = self.generate_query[hop](context=context, question=question).query\n",
+ " prev_queries.append(query)\n",
" passages = self.retrieve(query).passages\n",
" context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
+ " \n",
" pred = self.generate_answer(context=context, question=question)\n",
" pred = dspy.Prediction(context=context, answer=pred.answer)\n",
" return pred"
@@ -143,7 +168,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -155,6 +180,9 @@
" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
" self.max_hops = max_hops\n",
"\n",
+ " # for evaluating assertions only\n",
+ " self.passed_suggestions = 0\n",
+ "\n",
" def forward(self, question):\n",
" context = []\n",
" prev_queries = [question]\n",
@@ -176,6 +204,9 @@
" prev_queries.append(query)\n",
" passages = self.retrieve(query).passages\n",
" context = deduplicate(context + passages)\n",
+ " \n",
+ " if all_queries_distinct(prev_queries):\n",
+ " self.passed_suggestions += 1\n",
"\n",
" pred = self.generate_answer(context=context, question=question)\n",
" pred = dspy.Prediction(context=context, answer=pred.answer)\n",
@@ -184,28 +215,34 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"
+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"def evaluate(module):\n",
+ " module.passed_suggestions = 0\n",
+ "\n",
" retrieval_score = evaluate_on_hotpotqa(\n",
" module, metric=gold_passages_retrieved\n",
" )\n",
" \n",
+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",
+ "\n",
" accuracy_score = evaluate_on_hotpotqa(\n",
" module, metric=dspy.evaluate.answer_exact_match\n",
" )\n",
"\n",
+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",
+ "\n",
" print(f\"## Retrieval Score: {retrieval_score}\")\n",
" print(f\"## Accuracy Score: {accuracy_score}\")"
]