Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

reducing API changes

manishshettym1f4433b9be13
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index b33ea2694..15a744527 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -111,11 +111,10 @@ class BootstrapFewShot(Teleprompter):
break
if example_idx not in bootstrapped:
- success, suggest_failures = self._bootstrap_one_example(example, round_idx)
+ success = self._bootstrap_one_example(example, round_idx)
if success:
bootstrapped[example_idx] = True
- self.suggest_failures += suggest_failures
print(f'Bootstrapped {len(bootstrapped)} full traces after {example_idx+1} examples in round {round_idx}.')
print(f"# of suggestion failures during bootstrapping: {self.suggest_failures}")
@@ -188,8 +187,11 @@ class BootstrapFewShot(Teleprompter):
raise KeyError(f'Failed to find predictor {id(predictor)} {predictor} in {self.predictor2name}.') from e
name2traces[predictor_name].append(demo)
+
+ # increment the suggest_failures encountered during bootstrapping
+ self.suggest_failures += suggest_failures
- return success, suggest_failures
+ return success
def _train(self):
rng = random.Random(0)
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
index 36c3daa20..86d667841 100644
--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
@@ -2,10 +2,13 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
"import dspy\n",
"from dsp.utils import deduplicate\n",
"from dspy.datasets import HotPotQA\n",
@@ -18,7 +21,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -29,7 +32,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -41,7 +44,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -52,7 +55,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -79,7 +82,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -94,7 +97,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +120,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -143,7 +146,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
@@ -171,6 +174,7 @@
" validate_query_distinction_local(prev_queries, query),\n",
" \"Query should be distinct from: \"\n",
" + \"; \".join(f\"{i+1}) {q}\" for i, q in enumerate(prev_queries)),\n",
+ " is_metric=True,\n",
" )\n",
"\n",
" prev_queries.append(query)\n",
@@ -184,7 +188,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -193,7 +197,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -212,49 +216,400 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
- "# No Compilation + No Assertion\n",
- "baleen = SimplifiedBaleen()\n",
- "evaluate(baleen)\n"
+ "# # No Compilation + No Assertion\n",
+ "# baleen = SimplifiedBaleen()\n",
+ "# evaluate(baleen)\n"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
- "# No Compilation + Yes Assertion\n",
- "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
- "evaluate(baleen_with_assertions)"
+ "# # No Compilation + Yes Assertion\n",
+ "# baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
+ "# evaluate(baleen_with_assertions)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"# Yes Compilation + No Assertion\n",
- "baleen = SimplifiedBaleen()\n",
- "teleprompter = BootstrapFewShotWithRandomSearch(\n",
- " metric=validate_context_and_answer_and_hops,\n",
- " max_bootstrapped_demos=2,\n",
- " num_candidate_programs=6,\n",
- ")\n",
+ "# baleen = SimplifiedBaleen()\n",
+ "# teleprompter = BootstrapFewShotWithRandomSearch(\n",
+ "# metric=validate_context_and_answer_and_hops,\n",
+ "# max_bootstrapped_demos=2,\n",
+ "# num_candidate_programs=6,\n",
+ "# )\n",
"\n",
- "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
- "evaluate(compiled_baleen)"
+ "# compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
+ "# evaluate(compiled_baleen)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 21,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Going to sample between 1 and 2 traces per predictor.\n",
+ "Will attempt to train 6 candidate sets.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 524.07it/s]\n",
+ "/Users/manishs/Projects/dspy/dspy/evaluate/evaluate.py:130: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+ " df = df.applymap(truncate_cell)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0%)\n",
+ "Score: 43.0 for set: [0, 0, 0]\n",
+ "New best score: 43.0 for seed -3\n",
+ "Scores so far: [43.0]\n",
+ "Best score: 43.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:00<00:00, 514.39it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 42 / 100 (42.0%)\n",
+ "Score: 42.0 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0]\n",
+ "Best score: 43.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%|▏ | 4/300 [00:00<00:00, 1176.85it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 2 full traces after 5 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 4\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 45 / 100 (45.0): 100%|██████████| 100/100 [00:00<00:00, 517.62it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 45 / 100 (45.0%)\n",
+ "Score: 44.2 for set: [16, 16, 16]\n",
+ "New best score: 44.2 for seed -1\n",
+ "Scores so far: [43.0, 42.0, 44.2]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.51\n",
+ "Average of max per entry across top 3 scores: 0.54\n",
+ "Average of max per entry across top 5 scores: 0.54\n",
+ "Average of max per entry across top 8 scores: 0.54\n",
+ "Average of max per entry across top 9999 scores: 0.54\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 3/300 [00:00<00:00, 1135.03it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 2 full traces after 4 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 4\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0): 100%|██████████| 100/100 [00:00<00:00, 486.05it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 44 / 100 (44.0%)\n",
+ "Score: 43.2 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.56\n",
+ "Average of max per entry across top 8 scores: 0.56\n",
+ "Average of max per entry across top 9999 scores: 0.56\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 2/300 [00:00<00:00, 1088.72it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 468.94it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n",
+ "Score: 40.0 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.58\n",
+ "Average of max per entry across top 8 scores: 0.58\n",
+ "Average of max per entry across top 9999 scores: 0.58\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 2/300 [00:00<00:00, 1038.84it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 2\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0): 100%|██████████| 100/100 [00:00<00:00, 537.95it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 38 / 100 (38.0%)\n",
+ "Score: 37.6 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.58\n",
+ "Average of max per entry across top 8 scores: 0.58\n",
+ "Average of max per entry across top 9999 scores: 0.58\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%|▏ | 4/300 [00:00<00:00, 1060.71it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 5 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 1\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0): 100%|██████████| 100/100 [00:00<00:00, 428.19it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 37 / 100 (37.0%)\n",
+ "Score: 36.8 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.58\n",
+ "Average of max per entry across top 8 scores: 0.59\n",
+ "Average of max per entry across top 9999 scores: 0.59\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 3/300 [00:00<00:00, 1050.50it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 4 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 1\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0): 100%|██████████| 100/100 [00:00<00:00, 555.81it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 43 / 100 (43.0%)\n",
+ "Score: 42.8 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.57\n",
+ "Average of max per entry across top 8 scores: 0.6\n",
+ "Average of max per entry across top 9999 scores: 0.6\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 2/300 [00:00<00:00, 616.54it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 2 full traces after 3 examples in round 0.\n",
+ "# of suggestion failures during bootstrapping: 2\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0): 100%|██████████| 100/100 [00:00<00:00, 460.90it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40 / 100 (40.0%)\n",
+ "Score: 39.6 for set: [16, 16, 16]\n",
+ "Scores so far: [43.0, 42.0, 44.2, 43.2, 40.0, 37.6, 36.8, 42.8, 39.6]\n",
+ "Best score: 44.2\n",
+ "Average of max per entry across top 1 scores: 0.45\n",
+ "Average of max per entry across top 2 scores: 0.52\n",
+ "Average of max per entry across top 3 scores: 0.55\n",
+ "Average of max per entry across top 5 scores: 0.57\n",
+ "Average of max per entry across top 8 scores: 0.6\n",
+ "Average of max per entry across top 9999 scores: 0.61\n",
+ "9 candidate programs found.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 126 / 300 (42.0): 100%|██████████| 300/300 [00:00<00:00, 595.84it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 126 / 300 (42.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 150 / 300 (50.0): 100%|██████████| 300/300 [00:00<00:00, 618.44it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 150 / 300 (50.0%)\n",
+ "## Retrieval Score: 42.0\n",
+ "## Accuracy Score: 50.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
"source": [
"# Yes Compilation + Yes Assertion\n",
"baleen = SimplifiedBaleen()\n",
@@ -292,7 +647,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "version": "3.11.5"
}
},
"nbformat": 4,