DSPy: The framework for programming—not prompting—language models
Python
Revert back changes to intrinsic metrics
Shangyint7641d37eda15
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynbindex 36c3daa20..9e07b11bb 100644--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb@@ -52,7 +52,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 5,"metadata": {},"outputs": [],"source": [@@ -79,7 +79,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 6,"metadata": {},"outputs": [],"source": [@@ -94,7 +94,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 7,"metadata": {},"outputs": [],"source": [@@ -117,7 +117,22 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "def all_queries_distinct(prev_queries):\n",+ " query_distinct = True\n",+ " for i, query in enumerate(prev_queries):\n",+ " if validate_query_distinction_local(prev_queries[:i], query) == False:\n",+ " query_distinct = False\n",+ " break\n",+ " return query_distinct"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,"metadata": {},"outputs": [],"source": [@@ -130,12 +145,22 @@" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n"," self.max_hops = max_hops\n","\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n"," def forward(self, question):\n"," context = []\n",+ " prev_queries = [question]\n",+ "\n"," for hop in range(self.max_hops):\n"," query = self.generate_query[hop](context=context, question=question).query\n",+ " prev_queries.append(query)\n"," passages = self.retrieve(query).passages\n"," context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n",+ " \n"," pred = self.generate_answer(context=context, question=question)\n"," pred = dspy.Prediction(context=context, answer=pred.answer)\n"," return pred"@@ -143,7 +168,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 10,"metadata": {},"outputs": [],"source": [@@ -155,6 +180,9 @@" self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n"," self.max_hops = max_hops\n","\n",+ " # for evaluating assertions only\n",+ " self.passed_suggestions = 0\n",+ "\n"," def forward(self, question):\n"," context = []\n"," prev_queries = [question]\n",@@ -176,6 +204,9 @@" prev_queries.append(query)\n"," passages = self.retrieve(query).passages\n"," context = deduplicate(context + passages)\n",+ " \n",+ " if all_queries_distinct(prev_queries):\n",+ " self.passed_suggestions += 1\n","\n"," pred = self.generate_answer(context=context, question=question)\n"," pred = dspy.Prediction(context=context, answer=pred.answer)\n",@@ -184,28 +215,34 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 11,"metadata": {},"outputs": [],"source": [- "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"+ "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 12,"metadata": {},"outputs": [],"source": ["def evaluate(module):\n",+ " module.passed_suggestions = 0\n",+ "\n"," retrieval_score = evaluate_on_hotpotqa(\n"," module, metric=gold_passages_retrieved\n"," )\n"," \n",+ " suggestions_score = module.passed_suggestions / len(devset) * 100\n",+ "\n"," accuracy_score = evaluate_on_hotpotqa(\n"," module, metric=dspy.evaluate.answer_exact_match\n"," )\n","\n",+ " print(f\"## Suggestions Score: {suggestions_score}\")\n",+ "\n"," print(f\"## Retrieval Score: {retrieval_score}\")\n"," print(f\"## Accuracy Score: {accuracy_score}\")"]@@ -232,6 +269,15 @@"evaluate(baleen_with_assertions)"]},+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "max_bootstrapped_demos = 2"+ ]+ },{"cell_type": "code","execution_count": null,@@ -242,11 +288,11 @@"baleen = SimplifiedBaleen()\n","teleprompter = BootstrapFewShotWithRandomSearch(\n"," metric=validate_context_and_answer_and_hops,\n",- " max_bootstrapped_demos=2,\n",+ " max_bootstrapped_demos=max_bootstrapped_demos,\n"," num_candidate_programs=6,\n",")\n","\n",- "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",+ "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n","evaluate(compiled_baleen)"]},@@ -260,7 +306,7 @@"baleen = SimplifiedBaleen()\n","teleprompter = BootstrapFewShotWithRandomSearch(\n"," metric=validate_context_and_answer_and_hops,\n",- " max_bootstrapped_demos=2,\n",+ " max_bootstrapped_demos=max_bootstrapped_demos,\n"," num_candidate_programs=6,\n",")\n","compiled_baleen = teleprompter.compile(\n",@@ -270,9 +316,9 @@" ),\n"," teacher=baleen,\n"," trainset=trainset,\n",- " valset=devset[:100]\n",+ " valset=devset\n",")\n",- "evaluate(compiled_baleen)"+ "evaluate(compiled_baleen)\n"]}],@@ -292,7 +338,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.10.13"+ "version": "3.11.5"}},"nbformat": 4,