DSPy: The framework for programming—not prompting—language models
Python
feat(dspy): update remaining tests
Michael Jones476b86580997
diff --git a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py b/tests/dsp_LM/teleprompt/test_mipro_optimizer.pyindex 7536470b9..695dc2807 100644--- a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py+++ b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py@@ -210,28 +210,4 @@ def test_optimization_and_output_verification():test_input = "What is the capital of Spain?"prediction = optimized_student(input=test_input)- print("CORRECT ANSWER")- print(lm.get_convo(-1))-assert prediction.output == "Madrid"-- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- """- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- """- ),- },- ]diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pyindex 78cf69264..51fd88f4e 100644--- a/tests/predict/test_chain_of_thought.py+++ b/tests/predict/test_chain_of_thought.py@@ -6,32 +6,56 @@ from dspy.utils import DummyLMdef test_initialization_with_string_signature():- lm = DummyLM(["find the number after 1", "2"])+ lm = DummyLM(["[[ ## reasoning ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])dspy.settings.configure(lm=lm)predict = ChainOfThought("question -> answer")assert list(predict.extended_signature.output_fields.keys()) == [- "rationale",+ "reasoning","answer",]assert predict(question="What is 1+1?").answer == "2"for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)++ Your output fields are:+ 1. `reasoning` (str)+ 2. `answer` (str)++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## reasoning ## ]]+ {reasoning}++ [[ ## answer ## ]]+ {answer}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the fields `question`, produce the fields `answer`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""),},]diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.pyindex 99c7f8e3b..4ae612817 100644--- a/tests/predict/test_chain_of_thought_with_hint.py+++ b/tests/predict/test_chain_of_thought_with_hint.py@@ -17,30 +17,54 @@ def test_cot_with_no_hint():]assert predict(question="What is 1+1?").answer == "2"- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)++ Your output fields are:+ 1. `rationale` (str): ${produce the answer}. We ...+ 2. `answer` (str)++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## rationale ## ]]+ {rationale}++ [[ ## answer ## ]]+ {answer}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the fields `question`, produce the fields `answer`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ Respond with the corresponding output fields, starting with the field `rationale`, then `answer`, and then ending with the marker for `completed`."""),},]def test_cot_with_hint():- lm = DummyLM(["[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])+ lm = DummyLM(+ [+ "[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## hint ## ]]\nIs it helicopter?\n\n[[ ## answer ## ]]\n2"+ ]+ )dspy.settings.configure(lm=lm)predict = ChainOfThoughtWithHint("question -> answer")assert list(predict.extended_signature2.output_fields.keys()) == [@@ -52,21 +76,49 @@ def test_cot_with_hint():for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)++ Your output fields are:+ 1. `rationale` (str): ${produce the answer}. We ...+ 2. `hint` (str)+ 3. `answer` (str)++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## rationale ## ]]+ {rationale}++ [[ ## hint ## ]]+ {hint}++ [[ ## answer ## ]]+ {answer}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the fields `question`, produce the fields `answer`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ Respond with the corresponding output fields, starting with the field `rationale`, then `hint`, then `answer`, and then ending with the marker for `completed`."""),},]diff --git a/tests/predict/test_multi_chain_comparison.py b/tests/predict/test_multi_chain_comparison.pyindex 8c936a2d8..87e2af1a7 100644--- a/tests/predict/test_multi_chain_comparison.py+++ b/tests/predict/test_multi_chain_comparison.py@@ -30,7 +30,7 @@ def test_basic_example():# Call the MultiChainComparison on the completionsquestion = "What is the color of the sky?"- lm = DummyLM(["my rationale", "blue"])+ lm = DummyLM(["[[ ## rationale ## ]]\nmy rationale\n\n[[ ## answer ## ]]\nblue"])dspy.settings.configure(lm=lm)final_pred = compare_answers(completions, question=question)diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.pyindex 55a37e4b3..8ccb1970d 100644--- a/tests/predict/test_predict.py+++ b/tests/predict/test_predict.py@@ -41,28 +41,48 @@ def test_lm_after_dump_and_load_state():def test_call_method():predict_instance = Predict("input -> output")- lm = DummyLM(["test output"])+ lm = DummyLM(["[[ ## output ## ]]\ntest output"])dspy.settings.configure(lm=lm)result = predict_instance(input="test input")assert result.output == "test output"for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `input` (str)++ Your output fields are:+ 1. `output` (str)++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## input ## ]]+ {input}++ [[ ## output ## ]]+ {output}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the fields `input`, produce the fields `output`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## input ## ]]+ test input++ Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""),},]@@ -150,14 +170,16 @@ def test_typed_demos_after_dump_and_load_state():def test_forward_method():program = Predict("question -> answer")- dspy.settings.configure(lm=DummyLM([]))+ dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nNo more responses"]))result = program(question="What is 1+1?").answerassert result == "No more responses"def test_forward_method2():program = Predict("question -> answer1, answer2")- dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))+ dspy.settings.configure(+ lm=DummyLM(["[[ ## answer1 ## ]]\nmy first answer\n\n[[ ## answer2 ## ]]\nmy second answer"])+ )result = program(question="What is 1+1?")assert result.answer1 == "my first answer"assert result.answer2 == "my second answer"@@ -172,7 +194,7 @@ def test_config_management():def test_multi_output():program = Predict("question -> answer", n=2)- dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))+ dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nmy first answer", "[[ ## answer ## ]]\nmy second answer"]))results = program(question="What is 1+1?")assert results.completions.answer[0] == "my first answer"assert results.completions.answer[1] == "my second answer"@@ -183,8 +205,8 @@ def test_multi_output2():dspy.settings.configure(lm=DummyLM([- "my 0 answer\nAnswer 2: my 2 answer",- "my 1 answer\nAnswer 2: my 3 answer",+ "[[ ## answer1 ## ]]\nmy 0 answer\n\n[[ ## answer2 ## ]]\nmy 2 answer",+ "[[ ## answer1 ## ]]\nmy 1 answer\n\n[[ ## answer2 ## ]]\nmy 3 answer",],))@@ -215,27 +237,43 @@ def test_output_only():predictor = Predict(OutputOnlySignature)- lm = DummyLM(["short answer"])+ lm = DummyLM(["[[ ## output ## ]]\nshort answer"])dspy.settings.configure(lm=lm)assert predictor().output == "short answer"for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+++ Your output fields are:+ 1. `output` (str)++ All interactions will be structured in the following way, with the appropriate values filled in.++++ [[ ## output ## ]]+ {output}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the fields , produce the fields `output`."""),},{"role": "user",- "content": textwrap.dedent(- """\- """+ "content": (+ "Respond with the corresponding output fields, "+ "starting with the field `output`, and then ending with the marker for `completed`."),},]diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.pyindex 2980272f9..5bb654753 100644--- a/tests/predict/test_program_of_thought.py+++ b/tests/predict/test_program_of_thought.py@@ -11,96 +11,194 @@ class BasicQA(Signature):def test_pot_code_generation():- pot = ProgramOfThought(BasicQA)lm = DummyLM([- "Reason_A",- "```python\nresult = 1+1\n```",- "Reason_B",- "2",+ "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",+ "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## answer ## ]]\n2",])dspy.settings.configure(lm=lm)+ pot = ProgramOfThought(BasicQA)res = pot(question="What is 1+1?")assert res.answer == "2"- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")+assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)+ 2. `final_generated_code` (str): python code that answers the question+ 3. `code_output` (str): output of previously-generated python code++ Your output fields are:+ 1. `reasoning` (str)+ 2. `answer` (str): often between 1 and 5 words++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## final_generated_code ## ]]+ {final_generated_code}++ [[ ## code_output ## ]]+ {code_output}++ [[ ## reasoning ## ]]+ {reasoning}++ [[ ## answer ## ]]+ {answer}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ [[ ## final_generated_code ## ]]+ result = 1+1++ [[ ## code_output ## ]]+ 2++ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""),},]def test_pot_code_generation_with_error():- pot = ProgramOfThought(BasicQA)lm = DummyLM([- "Reason_A",- "```python\nresult = 1+0/0\n```",- "Reason_B", # Error: division by zero- "```python\nresult = 1+1\n```",- "Reason_C",- "2",+ "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+0/0\n```",+ "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",+ "[[ ## reasoning ## ]]\nReason_C\n\n[[ ## answer ## ]]\n2",])dspy.settings.configure(lm=lm)++ pot = ProgramOfThought(BasicQA)res = pot(question="What is 1+1?")assert res.answer == "2"# The first code example failed- for message in lm.get_convo(2)[0]:- print("----")- print(message["content"])- print("----")- assert lm.get_convo(2)[0] == [+ assert lm.get_convo(1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)+ 2. `previous_code` (str): previously-generated python code that errored+ 3. `error` (str): error message from previously-generated python code++ Your output fields are:+ 1. `reasoning` (str)+ 2. `generated_code` (str): python code that answers the question++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## previous_code ## ]]+ {previous_code}++ [[ ## error ## ]]+ {error}++ [[ ## reasoning ## ]]+ {reasoning}++ [[ ## generated_code ## ]]+ {generated_code}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ You are given `question`, `previous_code`, `error` due to an error in previous code.+ Your task is to correct the error and provide the new `generated_code`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ [[ ## previous_code ## ]]+ result = 1+0/0++ [[ ## error ## ]]+ division by zero++ Respond with the corresponding output fields, starting with the field `reasoning`, then `generated_code`, and then ending with the marker for `completed`."""),},]-- # The second code example succeeded- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")assert lm.get_convo(-1)[0] == [{"role": "system","content": textwrap.dedent("""\- """+ Your input fields are:+ 1. `question` (str)+ 2. `final_generated_code` (str): python code that answers the question+ 3. `code_output` (str): output of previously-generated python code++ Your output fields are:+ 1. `reasoning` (str)+ 2. `answer` (str): often between 1 and 5 words++ All interactions will be structured in the following way, with the appropriate values filled in.++ [[ ## question ## ]]+ {question}++ [[ ## final_generated_code ## ]]+ {final_generated_code}++ [[ ## code_output ## ]]+ {code_output}++ [[ ## reasoning ## ]]+ {reasoning}++ [[ ## answer ## ]]+ {answer}++ [[ ## completed ## ]]++ In adhering to this structure, your objective is:+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""),},{"role": "user","content": textwrap.dedent("""\- """+ [[ ## question ## ]]+ What is 1+1?++ [[ ## final_generated_code ## ]]+ result = 1+1++ [[ ## code_output ## ]]+ 2++ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""),},]diff --git a/tests/predict/test_react.py b/tests/predict/test_react.pyindex 9b187471b..8ee626aeb 100644--- a/tests/predict/test_react.py+++ b/tests/predict/test_react.py@@ -8,8 +8,7 @@ def test_example_no_tools():# Createa a simple dataset which the model will use with the Retrieve tool.lm = dspy.utils.DummyLM([- "Initial thoughts", # Thought_1- "Finish[blue]", # Action_1+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nFinish[blue]",])dspy.settings.configure(lm=lm, rm=dummy_rm())@@ -24,28 +23,13 @@ def test_example_no_tools():result = program(question=question)assert result.answer == "blue"- # For debugging- print("---")- for row in lm.history:- print(row["prompt"])- print("Response:", row["response"]["choices"][0]["text"])- print("---")-- assert lm.get_convo(-1).endswith(- "Question: What is the color of the sky?\n"- "Thought 1: Initial thoughts\n"- "Action 1: Finish[blue]"- )-def test_example_search():# Createa a simple dataset which the model will use with the Retrieve tool.lm = dspy.utils.DummyLM([- "Initial thoughts", # Thought_1- "Search[the color of the sky]", # Thought_1- "More thoughts", # Thought_2- "Finish[blue]", # Action_2+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nSearch[the color of the sky]",+ "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nFinish[blue]\n\n",])rm = dummy_rm(@@ -72,21 +56,6 @@ def test_example_search():result = program(question=question)assert result.answer == "blue"- # For debugging- print(lm.get_convo(-1))-- assert lm.get_convo(-1).endswith(- "Question: What is the color of the sky?\n\n"- "Thought 1: Initial thoughts\n\n"- "Action 1: Search[the color of the sky]\n\n"- "Observation 1:\n"- "[1] «We all know the color of the sky is blue.»\n"- "[2] «Somethng about the sky colors»\n"- "[3] «This sentence is completely irellevant to answer the question.»\n\n"- "Thought 2: More thoughts\n\n"- "Action 2: Finish[blue]"- )-class DummyTool1:name = "Tool1"@@ -122,12 +91,9 @@ class DummyTool2:def test_custom_tools():lm = dspy.utils.DummyLM([- "Initial thoughts",- "Tool1[foo]",- "More thoughts",- "Tool2[bar]",- "Even more thoughts",- "Finish[baz]",+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nTool1[foo]",+ "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nTool2[bar]",+ "[[ ## Thought_3 ## ]]\nEven more thoughts\n\n[[ ## Action_3 ## ]]\nFinish[baz]",])dspy.settings.configure(lm=lm)@@ -143,17 +109,6 @@ def test_custom_tools():# each tool should be called only onceassert tool1.num_calls == 1assert tool2.num_calls == 1- assert lm.get_convo(-1).endswith(- "Question: What is the color of the sky?\n\n"- "Thought 1: Initial thoughts\n\n"- "Action 1: Tool1[foo]\n\n"- "Observation 1: tool 1 output\n\n"- "Thought 2: More thoughts\n\n"- "Action 2: Tool2[bar]\n\n"- "Observation 2: tool 2 output\n\n"- "Thought 3: Even more thoughts\n\n"- "Action 3: Finish[baz]"- )def test_signature_instructions():diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.pyindex 5ceee8a62..16050e355 100644--- a/tests/predict/test_retry.py+++ b/tests/predict/test_retry.py@@ -14,7 +14,7 @@ def test_retry_simple():assert f"past_{field}" in retry_module.new_signature.input_fieldsassert "feedback" in retry_module.new_signature.input_fields- lm = DummyLM(["blue"])+ lm = DummyLM(["[[ ## answer ## ]]\nblue"])dspy.settings.configure(lm=lm)result = retry_module.forward(question="What color is the sky?",@@ -23,18 +23,10 @@ def test_retry_simple():)assert result.answer == "blue"- print(lm.get_convo(-1))- assert lm.get_convo(-1).endswith(- "Question: What color is the sky?\n\n"- "Previous Answer: red\n\n"- "Instructions: Try harder\n\n"- "Answer: blue"- )-def test_retry_forward_with_feedback():# First we make a mistake, then we fix it- lm = DummyLM(["red", "blue"])+ lm = DummyLM(["[[ ## answer ## ]]\nred", "[[ ## answer ## ]]\nblue"])dspy.settings.configure(lm=lm, trace=[])class SimpleModule(dspy.Module):@@ -58,18 +50,10 @@ def test_retry_forward_with_feedback():assert result.answer == "blue"- print(lm.get_convo(-1))- assert lm.get_convo(-1).endswith(- "Question: What color is the sky?\n\n"- "Previous Answer: red\n\n"- "Instructions: Please think harder\n\n"- "Answer: blue"- )-def test_retry_forward_with_typed_predictor():# First we make a mistake, then we fix it- lm = DummyLM(['{"answer":"red"}', '{"answer":"blue"}'])+ lm = DummyLM(['[[ ## output ## ]]\n{"answer":"red"}', '[[ ## output ## ]]\n{"answer":"blue"}'])dspy.settings.configure(lm=lm, trace=[])class AnswerQuestion(dspy.Signature):@@ -103,9 +87,3 @@ def test_retry_forward_with_typed_predictor():result = program(question="What color is the sky?")assert result.answer == "blue"- assert lm.get_convo(-1).endswith(- 'Input: {"question":"What color is the sky?"}\n\n'- 'Previous Output: {"answer":"red"}\n\n'- 'Instructions: Please think harder\n\n'- 'Output: {"answer":"blue"}'- )diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.pyindex 52f6f7477..292bafdf2 100644--- a/tests/signatures/test_signature.py+++ b/tests/signatures/test_signature.py@@ -182,6 +182,9 @@ def test_insantiating2():def test_multiline_instructions():+ lm = DummyLM(["[[ ## output ## ]]\nshort answer"])+ dspy.settings.configure(lm=lm)+class MySignature(Signature):"""First lineSecond line@@ -190,15 +193,8 @@ def test_multiline_instructions():output = OutputField()predictor = dspy.Predict(MySignature)-- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])- dspy.settings.configure(lm=lm)assert predictor().output == "short answer"- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")assert lm.get_convo(-1)[0] == [{"role": "system",@@ -219,7 +215,7 @@ def test_multiline_instructions():[[ ## completed ## ]]- In adhering to this structure, your objective is:+ In adhering to this structure, your objective is:First lineSecond lineThird line"""diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.pyindex 05431a632..ed3e61446 100644--- a/tests/teleprompt/test_bootstrap.py+++ b/tests/teleprompt/test_bootstrap.py@@ -82,7 +82,7 @@ def test_bootstrap_effectiveness():for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{@@ -105,7 +105,7 @@ def test_bootstrap_effectiveness():[[ ## completed ## ]]- In adhering to this structure, your objective is:+ In adhering to this structure, your objective is:Given the fields `input`, produce the fields `output`."""),},diff --git a/tests/teleprompt/test_copro_optimizer.py b/tests/teleprompt/test_copro_optimizer.pyindex 2618367ca..7f391f351 100644--- a/tests/teleprompt/test_copro_optimizer.py+++ b/tests/teleprompt/test_copro_optimizer.py@@ -123,7 +123,7 @@ def test_optimization_and_output_verification():for message in lm.get_convo(-1)[0]:print("----")- print(message["content"])+ print(textwrap.indent(message["content"], " "))print("----")assert lm.get_convo(-1)[0] == [{@@ -150,7 +150,7 @@ def test_optimization_and_output_verification():[[ ## completed ## ]]- In adhering to this structure, your objective is:+ In adhering to this structure, your objective is:Optimized Prompt"""),},diff --git a/tests/teleprompt/test_mipro_optimizer.py b/tests/teleprompt/test_mipro_optimizer.pydeleted file mode 100644index b586068f9..000000000--- a/tests/teleprompt/test_mipro_optimizer.py+++ /dev/null@@ -1,246 +0,0 @@-import re-import textwrap--import pytest--import dspy-from dsp.modules import LM-from dspy import Example-from dspy.teleprompt.signature_opt_bayesian import MIPRO-from dspy.utils.dummies import DummyLM---# Define a simple metric function for testing-def simple_metric(example, prediction, trace=None):- # Simplified metric for testing: true if prediction matches expected output- return example.output == prediction.output---# Some example data-capitals = {- "Germany": "Berlin",- "France": "Paris",- "Denmark": "Copenhagen",- "Sweden": "Stockholm",- "Norway": "Oslo",-}-# Not used for training data-extra_capitals = {- "Spain": "Madrid",- "Portugal": "Lisbon",- "Italy": "Rome",-}--# Example training and validation sets-trainset = [- Example(input="What is the color of the sky?", output="blue").with_inputs("input"),- Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),-] + [- Example(input=f"What is the capital of {country}?", output=capital).with_inputs("input")- for country, capital in capitals.items()-]---class ConditionalLM(LM):- def __init__(self):- super().__init__("conditional-lm")-- def basic_request(self, prompt, num_candidates=1, **kwargs):- # If we are in the "optimization" stage, we don't say much.- if prompt.endswith("Observations:"):- answer = " (*silence*)"- elif prompt.endswith("Proposed Instruction:"):- answer = " Input: "- elif prompt.endswith("Proposed Prefix For Output Field:"):- answer = " Output: "- elif prompt.endswith("Summary:"):- answer = " summarizing..."- else:- pairs = re.findall(r"Input: (.*?)\n(?:Reasoning:.*?\n)?Output: (.*?)\n", prompt, re.DOTALL)-- # breakpoint()- print("PROMPT:", prompt)- print("PAIRS:", pairs)-- last = re.search(r"Input: (.*)\nReasoning: (.*)$", prompt)- current_question = last.group(1)-- if match := re.match(r"What is the capital of (.*?)\?", current_question):- country = match.group(1)- # If we had a previous example of a question about a capital, the model- # has learned the format, and will answer with question correctly.- if any("capital" in question for question, _ in pairs):- answer = (capitals | extra_capitals)[country]- # Otherwise, it is confused and will answer with the country's name.- else:- answer = country- # For other questions, the model will answer with the last word of the question.- else:- answer = current_question.split()[-1]-- answer = "think deeply.\nOutput: " + answer-- RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m"- print("=== DummyLM ===")- print(prompt, end="")- print(f"{RED}{answer}{RESET}")- print("===")-- dummy_response = {"choices": []}- for _ in range(num_candidates):- dummy_response["choices"].append(- {- "text": answer,- "finish_reason": "done",- }- )-- # Simulate processing and storing the request and response.- history_entry = {- "prompt": prompt,- "response": dummy_response,- "kwargs": kwargs,- "raw_kwargs": kwargs,- }- self.history.append(history_entry)-- return dummy_response-- def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):- response = self.basic_request(prompt, **kwargs)- return [choice["text"] for choice in response["choices"]]-- def get_convo(self, index):- """get the prompt + anwer from the ith message"""- return self.history[index]["prompt"] + " " + self.history[index]["response"]["choices"][0]["text"]---def test_bayesian_signature_optimizer_initialization():- optimizer = MIPRO(metric=simple_metric, num_candidates=10, init_temperature=1.4, verbose=True, track_stats=True)- assert optimizer.metric == simple_metric, "Metric not correctly initialized"- assert optimizer.num_candidates == 10, "Incorrect 'num_candidates' parameter initialization"- assert optimizer.init_temperature == 1.4, "Initial temperature not correctly initialized"- assert optimizer.verbose is True, "Verbose flag not correctly initialized"- assert optimizer.track_stats is True, "Track stats flag not correctly initialized"---class SimpleModule(dspy.Module):- def __init__(self, signature):- super().__init__()- # SignatureOptimizer doesn't work with dspy.Predict- self.predictor = dspy.ChainOfThought(signature)-- def forward(self, **kwargs):- return self.predictor(**kwargs)---def test_signature_optimizer_optimization_process():- lm = ConditionalLM()- dspy.settings.configure(lm=lm)-- student = SimpleModule(signature="input -> output")-- optimizer = MIPRO(- metric=simple_metric,- num_candidates=10,- init_temperature=1.4,- verbose=False,- track_stats=False,- )-- # Adjustments: Include required parameters for the compile method- optimized_student = optimizer.compile(- student=student,- trainset=trainset,- num_trials=10,- max_bootstrapped_demos=3,- max_labeled_demos=5,- eval_kwargs={"num_threads": 1, "display_progress": False},- requires_permission_to_run=False,- )-- assert len(optimized_student.predictor.demos) == 5---def test_signature_optimizer_bad_lm():- dspy.settings.configure(lm=DummyLM([f"Optimized instruction {i}" for i in range(30)]))- student = SimpleModule(signature="input -> output")- optimizer = MIPRO(- metric=simple_metric,- num_candidates=10,- init_temperature=1.4,- verbose=False,- track_stats=False,- )-- # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DummyLM,- # which only outputs "Optimized instruction i" this means that none of the bootstrapped examples are successful,- # and therefore the set of examples that we're using to generate new prompts is empty- with pytest.raises(ValueError):- _optimized_student = optimizer.compile(- student=student,- trainset=trainset,- num_trials=10,- max_bootstrapped_demos=3,- max_labeled_demos=5,- eval_kwargs={"num_threads": 1, "display_progress": False},- requires_permission_to_run=False,- )---def test_optimization_and_output_verification():- # Make a language model that is always right, except on the last- # example in the train set.- lm = ConditionalLM()- dspy.settings.configure(lm=lm)-- optimizer = MIPRO(- metric=simple_metric,- num_candidates=10,- init_temperature=1.4,- verbose=False,- track_stats=True,- )-- student = SimpleModule("input -> output")-- # Compile the student with the optimizer- optimized_student = optimizer.compile(- student=student,- trainset=trainset,- num_trials=4,- max_bootstrapped_demos=2,- max_labeled_demos=3,- eval_kwargs={"num_threads": 1, "display_progress": False},- requires_permission_to_run=False,- )-- # Simulate calling the optimized student with a new input- test_input = "What is the capital of Spain?"- prediction = optimized_student(input=test_input)-- print("CORRECT ANSWER")- print(lm.get_convo(-1))-- assert prediction.output == "Madrid"-- for message in lm.get_convo(-1)[0]:- print("----")- print(message["content"])- print("----")- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- """- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- """- ),- },- ]