DSPy: The framework for programming—not prompting—language models
Python
reducing API changes
manishshettym1f4433b9be13
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex b33ea2694..15a744527 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -111,11 +111,10 @@ class BootstrapFewShot(Teleprompter):breakif example_idx not in bootstrapped:- success, suggest_failures = self._bootstrap_one_example(example, round_idx)+ success = self._bootstrap_one_example(example, round_idx)if success:bootstrapped[example_idx] = True- self.suggest_failures += suggest_failuresprint(f'Bootstrapped {len(bootstrapped)} full traces after {example_idx+1} examples in round {round_idx}.')print(f"# of suggestion failures during bootstrapping: {self.suggest_failures}")@@ -188,8 +187,11 @@ class BootstrapFewShot(Teleprompter):raise KeyError(f'Failed to find predictor {id(predictor)} {predictor} in {self.predictor2name}.') from ename2traces[predictor_name].append(demo)++ # increment the suggest_failures encountered during bootstrapping+ self.suggest_failures += suggest_failures- return success, suggest_failures+ return successdef _train(self):rng = random.Random(0)diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynbindex 36c3daa20..86d667841 100644--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb@@ -2,10 +2,13 @@"cells": [{"cell_type": "code",- "execution_count": 1,+ "execution_count": 2,"metadata": {},"outputs": [],"source": [+ "%load_ext autoreload\n",+ "%autoreload 2\n",+ "\n","import dspy\n","from dsp.utils import deduplicate\n","from dspy.datasets import HotPotQA\n",@@ -18,7 +21,7 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": 3,"metadata": {},"outputs": [],"source": [@@ -29,7 +32,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 4,"metadata": {},"outputs": [],"source": [@@ -41,7 +44,7 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": 5,"metadata": {},"outputs": [],"source": [@@ -52,7 +55,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 6,"metadata": {},"outputs": [],"source": [@@ -79,7 +82,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 7,"metadata": {},"outputs": [],"source": [@@ -94,7 +97,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 8,"metadata": {},"outputs": [],"source": [@@ -117,7 +120,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 9,"metadata": {},"outputs": [],"source": [@@ -143,7 +146,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 20,"metadata": {},"outputs": [],"source": [@@ -171,6 +174,7 @@" validate_query_distinction_local(prev_queries, query),\n"," \"Query should be distinct from: \"\n"," + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",+ " is_metric=True,\n"," )\n","\n"," prev_queries.append(query)\n",@@ -184,7 +188,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 11,"metadata": {},"outputs": [],"source": [@@ -193,7 +197,7 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 12,"metadata": {},"outputs": [],"source": [@@ -212,49 +216,400 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 13,"metadata": {},"outputs": [],"source": [- "# No Compilation + No Assertion\n",- "baleen = SimplifiedBaleen()\n",- "evaluate(baleen)\n"+ "# # No Compilation + No Assertion\n",+ "# baleen = SimplifiedBaleen()\n",+ "# evaluate(baleen)\n"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 14,"metadata": {},"outputs": [],"source": [- "# No Compilation + Yes Assertion\n",- "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",- "evaluate(baleen_with_assertions)"+ "# # No Compilation + Yes Assertion\n",+ "# baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",+ "# evaluate(baleen_with_assertions)"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 15,"metadata": {},"outputs": [],"source": ["# Yes Compilation + No Assertion\n",- "baleen = SimplifiedBaleen()\n",- "teleprompter = BootstrapFewShotWithRandomSearch(\n",- " metric=validate_context_and_answer_and_hops,\n",- " max_bootstrapped_demos=2,\n",- " num_candidate_programs=6,\n",- ")\n",+ "# baleen = SimplifiedBaleen()\n",+ "# teleprompter = BootstrapFewShotWithRandomSearch(\n",+ "# metric=validate_context_and_answer_and_hops,\n",+ "# max_bootstrapped_demos=2,\n",+ "# num_candidate_programs=6,\n",+ "# )\n","\n",- "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",- "evaluate(compiled_baleen)"+ "# compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",+ "# evaluate(compiled_baleen)"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 21,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Going to sample between 1 and 2 traces per predictor.\n",+ "Will attempt to train 6 candidate sets.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 524.07it/s]\n",+ "/Users/manishs/Projects/dspy/dspy/evaluate/evaluate.py:130: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",+ " df = df.applymap(truncate_cell)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0%)\n",+ "Score: 43.0 for set: [0, 0, 0]\n",+ "New best score: 43.0 for seed -3\n",+ "Scores so far: [43.0]\n",+ "Best score: 43.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:00<00:00, 514.39it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0%)\n",+ "Score: 42.0 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0]\n",+ "Best score: 43.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%|▏ | 4/300 [00:00<00:00, 1176.85it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 5 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 4\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:00<00:00, 517.62it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 45 / 100 (45.0%)\n",+ "Score: 44.2 for set: [16, 16, 16]\n",+ "New best score: 44.2 for seed -1\n",+ "Scores so far: [43.0, 42.0, 44.2]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.51\n",+ "Average of max per entry across top 3 scores: 0.54\n",+ "Average of max per entry across top 5 scores: 0.54\n",+ "Average of max per entry across top 8 scores: 0.54\n",+ "Average of max per entry across top 9999 scores: 0.54\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 3/300 [00:00<00:00, 1135.03it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 4 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 4\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:00<00:00, 486.05it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 44 / 100 (44.0%)\n",+ "Score: 43.2 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.56\n",+ "Average of max per entry across top 8 scores: 0.56\n",+ "Average of max per entry across top 9999 scores: 0.56\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 2/300 [00:00<00:00, 1088.72it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 468.94it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n",+ "Score: 40.0 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.58\n",+ "Average of max per entry across top 8 scores: 0.58\n",+ "Average of max per entry across top 9999 scores: 0.58\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 2/300 [00:00<00:00, 1038.84it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 2\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:00<00:00, 537.95it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 38 / 100 (38.0%)\n",+ "Score: 37.6 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.58\n",+ "Average of max per entry across top 8 scores: 0.58\n",+ "Average of max per entry across top 9999 scores: 0.58\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%|▏ | 4/300 [00:00<00:00, 1060.71it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 5 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 1\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:00<00:00, 428.19it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 37 / 100 (37.0%)\n",+ "Score: 36.8 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.58\n",+ "Average of max per entry across top 8 scores: 0.59\n",+ "Average of max per entry across top 9999 scores: 0.59\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 3/300 [00:00<00:00, 1050.50it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 4 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 1\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 555.81it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 43 / 100 (43.0%)\n",+ "Score: 42.8 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.57\n",+ "Average of max per entry across top 8 scores: 0.6\n",+ "Average of max per entry across top 9999 scores: 0.6\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 2/300 [00:00<00:00, 616.54it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 3 examples in round 0.\n",+ "# of suggestion failures during bootstrapping: 2\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 460.90it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 40 / 100 (40.0%)\n",+ "Score: 39.6 for set: [16, 16, 16]\n",+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8, 39.6]\n",+ "Best score: 44.2\n",+ "Average of max per entry across top 1 scores: 0.45\n",+ "Average of max per entry across top 2 scores: 0.52\n",+ "Average of max per entry across top 3 scores: 0.55\n",+ "Average of max per entry across top 5 scores: 0.57\n",+ "Average of max per entry across top 8 scores: 0.6\n",+ "Average of max per entry across top 9999 scores: 0.61\n",+ "9 candidate programs found.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 126 / 300 (42.0): 100%|██████████| 300/300 [00:00<00:00, 595.84it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 126 / 300 (42.0%)\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 150 / 300 (50.0): 100%|██████████| 300/300 [00:00<00:00, 618.44it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 150 / 300 (50.0%)\n",+ "## Retrieval Score: 42.0\n",+ "## Accuracy Score: 50.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ }+ ],"source": ["# Yes Compilation + Yes Assertion\n","baleen = SimplifiedBaleen()\n",@@ -292,7 +647,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.10.13"+ "version": "3.11.5"}},"nbformat": 4,