DSPy: The framework for programming—not prompting—language models
Python
update hotpotqa intrinsic metrics
Shangyint70db7ea2b0e6
diff --git a/hotpotqa_with_assertions.ipynb b/hotpotqa_with_assertions.ipynbindex 0d35b4da5..5c600a4c5 100644--- a/hotpotqa_with_assertions.ipynb+++ b/hotpotqa_with_assertions.ipynb@@ -2,7 +2,7 @@"cells": [{"cell_type": "code",- "execution_count": null,+ "execution_count": 1,"metadata": {},"outputs": [],"source": [@@ -18,7 +18,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 2,"metadata": {},"outputs": [],"source": [@@ -29,7 +29,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 3,"metadata": {},"outputs": [],"source": [@@ -41,7 +41,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 4,"metadata": {},"outputs": [],"source": [@@ -52,7 +52,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 5,"metadata": {},"outputs": [],"source": [@@ -79,7 +79,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 6,"metadata": {},"outputs": [],"source": [@@ -94,7 +94,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 7,"metadata": {},"outputs": [],"source": [@@ -117,7 +117,22 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "def all_queries_distinct(prev_queries):\n",+ " query_distinct = True\n",+ " for i, query in enumerate(prev_queries):\n",+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",+ " query_distinct = False\n",+ " break\n",+ " return query_distinct"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,"metadata": {},"outputs": [],"source": [@@ -130,12 +145,22 @@" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n"," self.max_hops = max_hops\n","\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n"," def forward(self, question):\n"," context = []\n",+ " prev_queries = [question]\n",+ "\n"," for hop in range(self.max_hops):\n"," query = self.generate_query[hop](context=context, question=question).query\n",+ " prev_queries.append(query)\n"," passages = self.retrieve(query).passages\n"," context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n",+ " \n"," pred = self.generate_answer(context=context, question=question)\n"," pred = dspy.Prediction(context=context, answer=pred.answer)\n"," return pred"@@ -143,7 +168,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 12,"metadata": {},"outputs": [],"source": [@@ -155,6 +180,9 @@" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n"," self.max_hops = max_hops\n","\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n"," def forward(self, question):\n"," context = []\n"," prev_queries = [question]\n",@@ -176,6 +204,9 @@" prev_queries.append(query)\n"," passages = self.retrieve(query).passages\n"," context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n","\n"," pred = self.generate_answer(context=context, question=question)\n"," pred = dspy.Prediction(context=context, answer=pred.answer)\n",@@ -184,28 +215,34 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 13,"metadata": {},"outputs": [],"source": [- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 14,"metadata": {},"outputs": [],"source": ["def evaluate(module):\n",+ " module.passed_suggestions = 0\n",+ "\n"," retrieval_score = evaluate_on_hotpotqa(\n"," module, metric=gold_passages_retrieved\n"," )\n"," \n",+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",+ "\n"," accuracy_score = evaluate_on_hotpotqa(\n"," module, metric=dspy.evaluate.answer_exact_match\n"," )\n","\n",+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",+ "\n"," print(f\"## Retrieval Score: {retrieval_score}\")\n"," print(f\"## Accuracy Score: {accuracy_score}\")"]