Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

assertion patches

Arnav Singhvib7435c175fb8
diff --cc docs/docs/cheatsheet.md
index 24e7e3ff8,24e7e3ff8..c067b6552
--- a/docs/docs/cheatsheet.md
+++ b/docs/docs/cheatsheet.md
@@@ -422,9 -422,9 +422,9 @@@ Other custom configurations are simila
### Including `dspy.Assert` and `dspy.Suggest` statements
```python
--dspy.Assert(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModuleSignature")
++dspy.Assert(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModule")
--dspy.Suggest(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModuleSignature")
++dspy.Suggest(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModule")
```
### Activating DSPy Program with Assertions
diff --cc examples/longformqa/longformqa_assertions.ipynb
index 9dd3e9e57,b96e8f56b..d179153ac
--- a/examples/longformqa/longformqa_assertions.ipynb
+++ b/examples/longformqa/longformqa_assertions.ipynb
@@@ -32,7 -32,7 +32,7 @@@
"metadata": {},
"source": [
"### 0] Setting Up\n",
-- "Let's begin by setting things up. The snippet below will retrieve the cached requests for the task."
++ "Let's begin by setting things up."
]
},
{
@@@ -41,28 -41,28 +41,16 @@@
"metadata": {},
"outputs": [],
"source": [
-- "!git clone https://huggingface.co/arnavs11/DSPy_LongFormQA\n",
-- "%cd DSPy_LongFormQA\n",
-- "!git checkout master\n",
-- "%cd ..\n",
"import os\n",
-- "repo_clone_path = '/content/DSPy_LongFormQA'\n",
-- "\n",
-- "# Check if '/content' is writable\n",
-- "if not os.access('/content', os.W_OK):\n",
-- " # If '/content' is not writable, choose an alternative directory\n",
-- " # Example: using a directory relative to the current working directory\n",
-- " repo_clone_path = os.path.join(os.getcwd(), 'DSPy_LongFormQA')\n",
-- "\n",
-- "# Set up the cache for this notebook\n",
-- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"
++ "import openai\n",
++ "openai.api_key = os.getenv('OPENAI_API_KEY')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
-- "We will also install **DSPy** if it's not there already."
++ "We will install **DSPy** if it's not there already."
]
},
{
@@@ -131,7 -131,7 +119,7 @@@
"source": [
"colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
"dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
-- "turbo = dspy.OpenAI(model='gpt-3.5-turbo-0613', max_tokens=500)\n",
++ "turbo = dspy.OpenAI(model='gpt-4o-mini', max_tokens=500)\n",
"dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
]
},
@@@ -150,7 -150,7 +138,7 @@@
"metadata": {},
"outputs": [],
"source": [
-- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0, keep_details=True)\n",
++ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0, keep_details=True)\n",
"trainset = [x.with_inputs('question') for x in dataset.train]\n",
"devset = [x.with_inputs('question') for x in dataset.dev]"
]
@@@ -415,11 -415,11 +403,21 @@@
},
{
"cell_type": "code",
-- "execution_count": null,
-- "metadata": {},
-- "outputs": [],
-- "source": [
-- "question = devset[28].question\n",
++ "execution_count": 41,
++ "metadata": {},
++ "outputs": [
++ {
++ "name": "stdout",
++ "output_type": "stream",
++ "text": [
++ "Question: What is the name of this region of Italy, referring to the medieval March of Ancona and nearby marches of Camerino and Fermo, where the comune Pollenza is located?\n",
++ "Predicted Paragraph: The region of Italy that refers to the medieval March of Ancona and nearby marches of Camerino and Fermo is called Marche. This name is derived from the plural form of \"marca,\" which originally denoted the frontier territories established during the Middle Ages, particularly the March of Ancona (Marche) (1). Today, Marche is recognized not only for its historical significance but also for its rich shoemaking tradition, producing some of the finest Italian footwear (1). Within this region lies the comune of Pollenza, located approximately 40 km southwest of Ancona, further illustrating the geographical and cultural significance of Marche (3).\n",
++ "Citation Faithfulness: False\n"
++ ]
++ }
++ ],
++ "source": [
++ "question = devset[15].question\n",
"pred = longformqa(question)\n",
"citation_faithfulness_score, _ = citation_faithfulness(None, pred, None)\n",
"\n",
@@@ -432,9 -432,9 +430,9 @@@
"cell_type": "markdown",
"metadata": {},
"source": [
-- "We can see that the generated paragraph does not properly include citations as intended. In some instances, it follows an incorrect citation format by not keeping the referenced source at before the period at the end of a sentence. In other instances, it does not provide citations for every 1-2 sentences as expected. \n",
++ "We can see that the generated paragraph does not properly include citations as intended in the format of \"[source]\". \n",
"\n",
-- "Additionally, we see that not all included citations are faithful to their preceding text (which can be a result of cited references in an incorrect format)."
++ "Additionally, we see that not all included citations are faithful to their preceding text."
]
},
{
@@@ -482,12 -482,12 +480,12 @@@
" context = deduplicate(context + passages)\n",
" pred = self.generate_cited_paragraph(context=context, question=question)\n",
" pred = dspy.Prediction(context=context, paragraph=pred.paragraph)\n",
- " dspy.Suggest(citations_check(pred.paragraph), \"Make sure every 1-2 sentences has citations. If any 1-2 sentences lack citations, add them in 'text... [x].' format.\", target_module=GenerateCitedParagraph)\n",
- " dspy.Suggest(citations_check(pred.paragraph), f\"Make sure every 1-2 sentences has citations. If any 1-2 sentences lack citations, add them in 'text... [x].' format.\", target_module=GenerateCitedParagraph)\n",
++ " dspy.Suggest(citations_check(pred.paragraph), \"Make sure every 1-2 sentences has citations. If any 1-2 sentences lack citations, add them in 'text... [x].' format.\", target_module=self.generate_cited_paragraph)\n",
" _, unfaithful_outputs = citation_faithfulness(None, pred, None)\n",
" if unfaithful_outputs:\n",
" unfaithful_pairs = [(output['text'], output['context']) for output in unfaithful_outputs]\n",
" for _, context in unfaithful_pairs:\n",
-- " dspy.Suggest(len(unfaithful_pairs) == 0, f\"Make sure your output is based on the following context: '{context}'.\", target_module=GenerateCitedParagraph)\n",
++ " dspy.Suggest(len(unfaithful_pairs) == 0, f\"Make sure your output is based on the following context: '{context}'.\", target_module=self.generate_cited_paragraph)\n",
" else:\n",
" return pred\n",
" return pred"
@@@ -534,11 -534,11 +532,21 @@@
},
{
"cell_type": "code",
-- "execution_count": null,
-- "metadata": {},
-- "outputs": [],
-- "source": [
-- "question = devset[28].question\n",
++ "execution_count": 42,
++ "metadata": {},
++ "outputs": [
++ {
++ "name": "stdout",
++ "output_type": "stream",
++ "text": [
++ "Question: What is the name of this region of Italy, referring to the medieval March of Ancona and nearby marches of Camerino and Fermo, where the comune Pollenza is located?\n",
++ "Predicted Paragraph: The region of Italy that refers to the medieval March of Ancona and nearby marches of Camerino and Fermo is called Marche [1]. This name is derived from the plural form of \"marca,\" which originally denoted the frontier territories established during the Middle Ages, particularly the March of Ancona [2]. Today, Marche is recognized not only for its historical significance but also for its rich shoemaking tradition, producing some of the finest Italian footwear [1]. Within this region lies the comune of Pollenza, located approximately 40 km southwest of Ancona and about 9 km southwest of Macerata, further illustrating the geographical and cultural significance of Marche [3].\n",
++ "Citation Faithfulness: True\n"
++ ]
++ }
++ ],
++ "source": [
++ "question = devset[15].question\n",
"pred = longformqa_with_assertions(question)\n",
"citation_faithfulness_score, _ = citation_faithfulness(None, pred, None)\n",
"\n",
@@@ -551,7 -551,7 +559,7 @@@
"cell_type": "markdown",
"metadata": {},
"source": [
-- "We now see that both computational constraints are indeed met. Every 1-2 sentences includes a citation and from our citation_faithfulness check, we see that each reference is also faithful to its preceding text. "
++ "We now see that both computational constraints are indeed met. Every 1-2 sentences includes a citation and from our `citation_faithfulness` check, we see that each reference is also faithful to its preceding text. "
]
},
{
@@@ -575,7 -575,7 +583,7 @@@
"source": [
"longformqa = LongFormQA()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "cited_longformqa = teleprompter.compile(student = longformqa, teacher = longformqa, trainset=trainset, valset=devset[:100])\n",
++ "cited_longformqa = teleprompter.compile(student = longformqa, teacher = longformqa, trainset=trainset, valset=devset[:25])\n",
"evaluate(cited_longformqa)"
]
},
@@@ -596,7 -596,7 +604,7 @@@
"source": [
"longformqa = LongFormQA()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "cited_longformqa_teacher = teleprompter.compile(student=longformqa, teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",
++ "cited_longformqa_teacher = teleprompter.compile(student=longformqa, teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:25])\n",
"evaluate(cited_longformqa_teacher)"
]
},
@@@ -615,7 -615,7 +623,7 @@@
"source": [
"longformqa = LongFormQA()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "cited_longformqa_student_teacher = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",
++ "cited_longformqa_student_teacher = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:25])\n",
"evaluate(cited_longformqa_student_teacher)"
]
}
diff --cc examples/qa/hotpot/hotpotqa_with_assertions.ipynb
index 514447750,7caa7f395..8bf62b6c4
--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
@@@ -35,7 -35,7 +35,7 @@@
"source": [
"colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
"dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
-- "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=500)\n",
++ "turbo = dspy.OpenAI(model='gpt-4o-mini', max_tokens=500)\n",
"dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
]
},
@@@ -45,7 -45,7 +45,7 @@@
"metadata": {},
"outputs": [],
"source": [
-- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0)\n",
++ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0, keep_details=True)\n",
"trainset = [x.with_inputs('question') for x in dataset.train]\n",
"devset = [x.with_inputs('question') for x in dataset.dev]"
]
diff --cc examples/quiz/quiz_assertions.ipynb
index 086d8268b,4e17c1753..b92263064
--- a/examples/quiz/quiz_assertions.ipynb
+++ b/examples/quiz/quiz_assertions.ipynb
@@@ -30,21 -30,21 +30,9 @@@
"metadata": {},
"outputs": [],
"source": [
-- "!git clone https://huggingface.co/arnavs11/DSPy_QuizGen_Cache\n",
-- "%cd DSPy_QuizGen_Cache/\n",
-- "!git checkout master\n",
-- "%cd ..\n",
"import os\n",
-- "repo_clone_path = '/content/DSPy_QuizGen_Cache'\n",
-- "\n",
-- "# Check if '/content' is writable\n",
-- "if not os.access('/content', os.W_OK):\n",
-- " # If '/content' is not writable, choose an alternative directory\n",
-- " # Example: using a directory relative to the current working directory\n",
-- " repo_clone_path = os.path.join(os.getcwd(), 'DSPy_QuizGen_Cache')\n",
-- "\n",
-- "# Set up the cache for this notebook\n",
-- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"
++ "import openai\n",
++ "openai.api_key = os.getenv('OPENAI_API_KEY')"
]
},
{
@@@ -95,7 -96,7 +83,7 @@@
"source": [
"colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
"dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
-- "turbo = dspy.OpenAI(model='gpt-3.5-turbo-0613', max_tokens=500)\n",
++ "turbo = dspy.OpenAI(model='gpt-4o-mini', max_tokens=500)\n",
"dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
]
},
@@@ -105,7 -106,7 +93,7 @@@
"metadata": {},
"outputs": [],
"source": [
-- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0, keep_details=True)\n",
++ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0, keep_details=True)\n",
"trainset = [x.with_inputs('question', 'answer') for x in dataset.train]\n",
"devset = [x.with_inputs('question', 'answer') for x in dataset.dev]"
]
@@@ -341,11 -342,11 +329,11 @@@
"\n",
" def forward(self, question, answer):\n",
" choice_string = self.generate_choices(question=question, correct_answer=answer, number_of_choices=number_of_choices).answer_choices\n",
-- " dspy.Suggest(format_checker(choice_string), \"The format of the answer choices should be in JSON format. Please revise accordingly.\", target_module=GenerateAnswerChoices)\n",
-- " dspy.Suggest(is_correct_answer_included(answer, choice_string), \"The answer choices do not include the correct answer to the question. Please revise accordingly.\", target_module=GenerateAnswerChoices)\n",
++ " dspy.Suggest(format_checker(choice_string), \"The format of the answer choices should be in JSON format. Please revise accordingly.\", target_module=self.generate_choices)\n",
++ " dspy.Suggest(is_correct_answer_included(answer, choice_string), \"The answer choices do not include the correct answer to the question. Please revise accordingly.\", target_module=self.generate_choices)\n",
" plausibility_question = \"Are the distractors in the answer choices plausible and not easily identifiable as incorrect?\"\n",
" plausibility_assessment = dspy.Predict(AssessQuizChoices)(question=question, answer_choices=choice_string, assessment_question=plausibility_question)\n",
-- " dspy.Suggest(is_plausibility_yes(plausibility_assessment.assessment_answer), \"The answer choices are not plausible distractors or are too easily identifiable as incorrect. Please revise to provide more challenging and plausible distractors.\", target_module=GenerateAnswerChoices)\n",
++ " dspy.Suggest(is_plausibility_yes(plausibility_assessment.assessment_answer), \"The answer choices are not plausible distractors or are too easily identifiable as incorrect. Please revise to provide more challenging and plausible distractors.\", target_module=self.generate_choices)\n",
" return dspy.Prediction(choices = choice_string)\n",
"\n",
"number_of_choices = '4'\n",
@@@ -428,7 -429,7 +416,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "compiled_quiz_generator = teleprompter.compile(student = quiz_generator, teacher = quiz_generator, trainset=trainset, valset=devset[:100])\n",
++ "compiled_quiz_generator = teleprompter.compile(student = quiz_generator, teacher = quiz_generator, trainset=trainset, valset=devset[:25])\n",
"\n",
"for metric in metrics:\n",
" evaluate = Evaluate(metric=metric, devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
@@@ -453,7 -454,7 +441,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "compiled_with_assertions_quiz_generator = teleprompter.compile(student=quiz_generator, teacher = quiz_generator_with_assertions, trainset=trainset, valset=devset[:100])\n",
++ "compiled_with_assertions_quiz_generator = teleprompter.compile(student=quiz_generator, teacher = quiz_generator_with_assertions, trainset=trainset, valset=devset[:25])\n",
"\n",
"\n",
"for metric in metrics:\n",
@@@ -468,7 -469,7 +456,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "compiled_quiz_generator_with_assertions = teleprompter.compile(student=quiz_generator_with_assertions, teacher = quiz_generator_with_assertions, trainset=trainset, valset=devset[:100])\n",
++ "compiled_quiz_generator_with_assertions = teleprompter.compile(student=quiz_generator_with_assertions, teacher = quiz_generator_with_assertions, trainset=trainset, valset=devset[:25])\n",
"\n",
"for metric in metrics:\n",
" evaluate = Evaluate(metric=metric, devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
diff --cc examples/tweets/tweets_assertions.ipynb
index 5f96f592b,39c328096..bde086aeb
--- a/examples/tweets/tweets_assertions.ipynb
+++ b/examples/tweets/tweets_assertions.ipynb
@@@ -30,21 -30,21 +30,9 @@@
"metadata": {},
"outputs": [],
"source": [
-- "!git clone https://huggingface.co/arnavs11/DSPy_TweetGen_Cache\n",
-- "%cd DSPy_TweetGen_Cache/\n",
-- "!git checkout master\n",
-- "%cd ..\n",
"import os\n",
-- "repo_clone_path = '/content/DSPy_TweetGen_Cache'\n",
-- "\n",
-- "# Check if '/content' is writable\n",
-- "if not os.access('/content', os.W_OK):\n",
-- " # If '/content' is not writable, choose an alternative directory\n",
-- " # Example: using a directory relative to the current working directory\n",
-- " repo_clone_path = os.path.join(os.getcwd(), 'DSPy_TweetGen_Cache')\n",
-- "\n",
-- "# Set up the cache for this notebook\n",
-- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"
++ "import openai\n",
++ "openai.api_key = os.getenv('OPENAI_API_KEY')"
]
},
{
@@@ -96,7 -97,7 +84,7 @@@
"source": [
"colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
"dspy.settings.configure(rm=colbertv2_wiki17_abstracts)\n",
-- "turbo = dspy.OpenAI(model='gpt-3.5-turbo-0613', max_tokens=500)\n",
++ "turbo = dspy.OpenAI(model='gpt-4o-mini', max_tokens=500)\n",
"dspy.settings.configure(lm=turbo, trace=[], temperature=0.7)"
]
},
@@@ -106,7 -107,7 +94,7 @@@
"metadata": {},
"outputs": [],
"source": [
-- "dataset = HotPotQA(train_seed=1, train_size=300, eval_seed=2023, dev_size=300, test_size=0, keep_details=True)\n",
++ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0, keep_details=True)\n",
"trainset = [x.with_inputs('question', 'answer') for x in dataset.train]\n",
"devset = [x.with_inputs('question', 'answer') for x in dataset.dev]"
]
@@@ -386,15 -387,15 +374,15 @@@
" passages = retrieve(query).passages\n",
" context = deduplicate(context + passages)\n",
" generated_tweet = self.generate_tweet(question=question, context=context).tweet\n",
- " dspy.Suggest(has_no_hashtags(generated_tweet), \"Please revise the tweet to remove hashtag phrases following it.\", target_module=GenerateTweet)\n",
- " dspy.Suggest(has_no_hashtags(generated_tweet), f\"Please revise the tweet to remove hashtag phrases following it.\", target_module=GenerateTweet)\n",
-- " dspy.Suggest(is_within_length_limit(generated_tweet, 280), f\"Please ensure the tweet is within {280} characters.\", target_module=GenerateTweet)\n",
-- " dspy.Suggest(has_correct_answer(generated_tweet, answer), \"The tweet does not include the correct answer to the question. Please revise accordingly.\", target_module=GenerateTweet)\n",
++ " dspy.Suggest(has_no_hashtags(generated_tweet), \"Please revise the tweet to remove hashtag phrases following it.\", target_module=self.generate_tweet)\n",
++ " dspy.Suggest(is_within_length_limit(generated_tweet, 280), f\"Please ensure the tweet is within {280} characters.\", target_module=self.generate_tweet)\n",
++ " dspy.Suggest(has_correct_answer(generated_tweet, answer), \"The tweet does not include the correct answer to the question. Please revise accordingly.\", target_module=self.generate_tweet)\n",
" engaging_question = \"Does the assessed text make for a self-contained, engaging tweet? Say no if it is not engaging.\"\n",
" engaging_assessment = dspy.Predict(AssessTweet)(context=context, assessed_text=generated_tweet, assessment_question=engaging_question)\n",
-- " dspy.Suggest(is_assessment_yes(engaging_assessment.assessment_answer), \"The text is not engaging enough. Please revise to make it more captivating.\", target_module=GenerateTweet)\n",
++ " dspy.Suggest(is_assessment_yes(engaging_assessment.assessment_answer), \"The text is not engaging enough. Please revise to make it more captivating.\", target_module=self.generate_tweet)\n",
" faithful_question = \"Is the assessed text grounded in the context? Say no if it includes significant facts not in the context.\"\n",
" faithful_assessment = dspy.Predict(AssessTweet)(context='N/A', assessed_text=generated_tweet, assessment_question=faithful_question)\n",
-- " dspy.Suggest(is_assessment_yes(faithful_assessment.assessment_answer), \"The text contains unfaithful elements or significant facts not in the context. Please revise for accuracy.\", target_module=GenerateTweet)\n",
++ " dspy.Suggest(is_assessment_yes(faithful_assessment.assessment_answer), \"The text contains unfaithful elements or significant facts not in the context. Please revise for accuracy.\", target_module=self.generate_tweet)\n",
" return dspy.Prediction(generated_tweet=generated_tweet, context=context)\n",
"\n",
"tweeter_with_assertions = assert_transform_module(TweeterWithAssertions().map_named_predictors(Retry), backtrack_handler) "
@@@ -477,7 -478,7 +465,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "compiled_tweeter = teleprompter.compile(student = tweeter, teacher = tweeter, trainset=trainset, valset=devset[:100])\n",
++ "compiled_tweeter = teleprompter.compile(student = tweeter, teacher = tweeter, trainset=trainset, valset=devset[:25])\n",
"\n",
"for metric in metrics:\n",
" evaluate = Evaluate(metric=metric, devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
@@@ -502,7 -503,7 +490,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
-- "compiled_with_assertions_tweeter = teleprompter.compile(student=tweeter, teacher = tweeter_with_assertions, trainset=trainset, valset=devset[:100])\n",
++ "compiled_with_assertions_tweeter = teleprompter.compile(student=tweeter, teacher = tweeter_with_assertions, trainset=trainset, valset=devset[:25])\n",
"\n",
"\n",
"for metric in metrics:\n",
@@@ -517,7 -518,7 +505,7 @@@
"outputs": [],
"source": [
"teleprompter = BootstrapFewShotWithRandomSearch(metric = overall_metric, max_bootstrapped_demos=2, num_candidate_programs=6, num_threads=1)\n",
-- "compiled_tweeter_with_assertions = teleprompter.compile(student=tweeter_with_assertions, teacher = tweeter_with_assertions, trainset=trainset, valset=devset[:100])\n",
++ "compiled_tweeter_with_assertions = teleprompter.compile(student=tweeter_with_assertions, teacher = tweeter_with_assertions, trainset=trainset, valset=devset[:25])\n",
"\n",
"for metric in metrics:\n",
" evaluate = Evaluate(metric=metric, devset=devset, num_threads=1, display_progress=True, display_table=5)\n",