Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): update remaining tests

Michael Jones476b86580997
diff --git a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
index 7536470b9..695dc2807 100644
--- a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
+++ b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
@@ -210,28 +210,4 @@ def test_optimization_and_output_verification():
test_input = "What is the capital of Spain?"
prediction = optimized_student(input=test_input)
- print("CORRECT ANSWER")
- print(lm.get_convo(-1))
-
assert prediction.output == "Madrid"
-
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- """
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- """
- ),
- },
- ]
diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.py
index 78cf69264..51fd88f4e 100644
--- a/tests/predict/test_chain_of_thought.py
+++ b/tests/predict/test_chain_of_thought.py
@@ -6,32 +6,56 @@ from dspy.utils import DummyLM
def test_initialization_with_string_signature():
- lm = DummyLM(["find the number after 1", "2"])
+ lm = DummyLM(["[[ ## reasoning ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])
dspy.settings.configure(lm=lm)
predict = ChainOfThought("question -> answer")
assert list(predict.extended_signature.output_fields.keys()) == [
- "rationale",
+ "reasoning",
"answer",
]
assert predict(question="What is 1+1?").answer == "2"
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+
+ Your output fields are:
+ 1. `reasoning` (str)
+ 2. `answer` (str)
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## reasoning ## ]]
+ {reasoning}
+
+ [[ ## answer ## ]]
+ {answer}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the fields `question`, produce the fields `answer`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
),
},
]
diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.py
index 99c7f8e3b..4ae612817 100644
--- a/tests/predict/test_chain_of_thought_with_hint.py
+++ b/tests/predict/test_chain_of_thought_with_hint.py
@@ -17,30 +17,54 @@ def test_cot_with_no_hint():
]
assert predict(question="What is 1+1?").answer == "2"
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+
+ Your output fields are:
+ 1. `rationale` (str): ${produce the answer}. We ...
+ 2. `answer` (str)
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## rationale ## ]]
+ {rationale}
+
+ [[ ## answer ## ]]
+ {answer}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the fields `question`, produce the fields `answer`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ Respond with the corresponding output fields, starting with the field `rationale`, then `answer`, and then ending with the marker for `completed`."""
),
},
]
def test_cot_with_hint():
- lm = DummyLM(["[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])
+ lm = DummyLM(
+ [
+ "[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## hint ## ]]\nIs it helicopter?\n\n[[ ## answer ## ]]\n2"
+ ]
+ )
dspy.settings.configure(lm=lm)
predict = ChainOfThoughtWithHint("question -> answer")
assert list(predict.extended_signature2.output_fields.keys()) == [
@@ -52,21 +76,49 @@ def test_cot_with_hint():
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+
+ Your output fields are:
+ 1. `rationale` (str): ${produce the answer}. We ...
+ 2. `hint` (str)
+ 3. `answer` (str)
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## rationale ## ]]
+ {rationale}
+
+ [[ ## hint ## ]]
+ {hint}
+
+ [[ ## answer ## ]]
+ {answer}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the fields `question`, produce the fields `answer`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ Respond with the corresponding output fields, starting with the field `rationale`, then `hint`, then `answer`, and then ending with the marker for `completed`."""
),
},
]
diff --git a/tests/predict/test_multi_chain_comparison.py b/tests/predict/test_multi_chain_comparison.py
index 8c936a2d8..87e2af1a7 100644
--- a/tests/predict/test_multi_chain_comparison.py
+++ b/tests/predict/test_multi_chain_comparison.py
@@ -30,7 +30,7 @@ def test_basic_example():
# Call the MultiChainComparison on the completions
question = "What is the color of the sky?"
- lm = DummyLM(["my rationale", "blue"])
+ lm = DummyLM(["[[ ## rationale ## ]]\nmy rationale\n\n[[ ## answer ## ]]\nblue"])
dspy.settings.configure(lm=lm)
final_pred = compare_answers(completions, question=question)
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index 55a37e4b3..8ccb1970d 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -41,28 +41,48 @@ def test_lm_after_dump_and_load_state():
def test_call_method():
predict_instance = Predict("input -> output")
- lm = DummyLM(["test output"])
+ lm = DummyLM(["[[ ## output ## ]]\ntest output"])
dspy.settings.configure(lm=lm)
result = predict_instance(input="test input")
assert result.output == "test output"
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `input` (str)
+
+ Your output fields are:
+ 1. `output` (str)
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## input ## ]]
+ {input}
+
+ [[ ## output ## ]]
+ {output}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the fields `input`, produce the fields `output`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## input ## ]]
+ test input
+
+ Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
),
},
]
@@ -150,14 +170,16 @@ def test_typed_demos_after_dump_and_load_state():
def test_forward_method():
program = Predict("question -> answer")
- dspy.settings.configure(lm=DummyLM([]))
+ dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nNo more responses"]))
result = program(question="What is 1+1?").answer
assert result == "No more responses"
def test_forward_method2():
program = Predict("question -> answer1, answer2")
- dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))
+ dspy.settings.configure(
+ lm=DummyLM(["[[ ## answer1 ## ]]\nmy first answer\n\n[[ ## answer2 ## ]]\nmy second answer"])
+ )
result = program(question="What is 1+1?")
assert result.answer1 == "my first answer"
assert result.answer2 == "my second answer"
@@ -172,7 +194,7 @@ def test_config_management():
def test_multi_output():
program = Predict("question -> answer", n=2)
- dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))
+ dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nmy first answer", "[[ ## answer ## ]]\nmy second answer"]))
results = program(question="What is 1+1?")
assert results.completions.answer[0] == "my first answer"
assert results.completions.answer[1] == "my second answer"
@@ -183,8 +205,8 @@ def test_multi_output2():
dspy.settings.configure(
lm=DummyLM(
[
- "my 0 answer\nAnswer 2: my 2 answer",
- "my 1 answer\nAnswer 2: my 3 answer",
+ "[[ ## answer1 ## ]]\nmy 0 answer\n\n[[ ## answer2 ## ]]\nmy 2 answer",
+ "[[ ## answer1 ## ]]\nmy 1 answer\n\n[[ ## answer2 ## ]]\nmy 3 answer",
],
)
)
@@ -215,27 +237,43 @@ def test_output_only():
predictor = Predict(OutputOnlySignature)
- lm = DummyLM(["short answer"])
+ lm = DummyLM(["[[ ## output ## ]]\nshort answer"])
dspy.settings.configure(lm=lm)
assert predictor().output == "short answer"
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+
+
+ Your output fields are:
+ 1. `output` (str)
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+
+
+ [[ ## output ## ]]
+ {output}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the fields , produce the fields `output`."""
),
},
{
"role": "user",
- "content": textwrap.dedent(
- """\
- """
+ "content": (
+ "Respond with the corresponding output fields, "
+ "starting with the field `output`, and then ending with the marker for `completed`."
),
},
]
diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.py
index 2980272f9..5bb654753 100644
--- a/tests/predict/test_program_of_thought.py
+++ b/tests/predict/test_program_of_thought.py
@@ -11,96 +11,194 @@ class BasicQA(Signature):
def test_pot_code_generation():
- pot = ProgramOfThought(BasicQA)
lm = DummyLM(
[
- "Reason_A",
- "```python\nresult = 1+1\n```",
- "Reason_B",
- "2",
+ "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",
+ "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## answer ## ]]\n2",
]
)
dspy.settings.configure(lm=lm)
+ pot = ProgramOfThought(BasicQA)
res = pot(question="What is 1+1?")
assert res.answer == "2"
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
+
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+ 2. `final_generated_code` (str): python code that answers the question
+ 3. `code_output` (str): output of previously-generated python code
+
+ Your output fields are:
+ 1. `reasoning` (str)
+ 2. `answer` (str): often between 1 and 5 words
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## final_generated_code ## ]]
+ {final_generated_code}
+
+ [[ ## code_output ## ]]
+ {code_output}
+
+ [[ ## reasoning ## ]]
+ {reasoning}
+
+ [[ ## answer ## ]]
+ {answer}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ [[ ## final_generated_code ## ]]
+ result = 1+1
+
+ [[ ## code_output ## ]]
+ 2
+
+ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
),
},
]
def test_pot_code_generation_with_error():
- pot = ProgramOfThought(BasicQA)
lm = DummyLM(
[
- "Reason_A",
- "```python\nresult = 1+0/0\n```",
- "Reason_B", # Error: division by zero
- "```python\nresult = 1+1\n```",
- "Reason_C",
- "2",
+ "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+0/0\n```",
+ "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",
+ "[[ ## reasoning ## ]]\nReason_C\n\n[[ ## answer ## ]]\n2",
]
)
dspy.settings.configure(lm=lm)
+
+ pot = ProgramOfThought(BasicQA)
res = pot(question="What is 1+1?")
assert res.answer == "2"
# The first code example failed
- for message in lm.get_convo(2)[0]:
- print("----")
- print(message["content"])
- print("----")
- assert lm.get_convo(2)[0] == [
+ assert lm.get_convo(1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+ 2. `previous_code` (str): previously-generated python code that errored
+ 3. `error` (str): error message from previously-generated python code
+
+ Your output fields are:
+ 1. `reasoning` (str)
+ 2. `generated_code` (str): python code that answers the question
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## previous_code ## ]]
+ {previous_code}
+
+ [[ ## error ## ]]
+ {error}
+
+ [[ ## reasoning ## ]]
+ {reasoning}
+
+ [[ ## generated_code ## ]]
+ {generated_code}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ You are given `question`, `previous_code`, `error` due to an error in previous code.
+ Your task is to correct the error and provide the new `generated_code`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ [[ ## previous_code ## ]]
+ result = 1+0/0
+
+ [[ ## error ## ]]
+ division by zero
+
+ Respond with the corresponding output fields, starting with the field `reasoning`, then `generated_code`, and then ending with the marker for `completed`."""
),
},
]
-
- # The second code example succeeded
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
"content": textwrap.dedent(
"""\
- """
+ Your input fields are:
+ 1. `question` (str)
+ 2. `final_generated_code` (str): python code that answers the question
+ 3. `code_output` (str): output of previously-generated python code
+
+ Your output fields are:
+ 1. `reasoning` (str)
+ 2. `answer` (str): often between 1 and 5 words
+
+ All interactions will be structured in the following way, with the appropriate values filled in.
+
+ [[ ## question ## ]]
+ {question}
+
+ [[ ## final_generated_code ## ]]
+ {final_generated_code}
+
+ [[ ## code_output ## ]]
+ {code_output}
+
+ [[ ## reasoning ## ]]
+ {reasoning}
+
+ [[ ## answer ## ]]
+ {answer}
+
+ [[ ## completed ## ]]
+
+ In adhering to this structure, your objective is:
+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""
),
},
{
"role": "user",
"content": textwrap.dedent(
"""\
- """
+ [[ ## question ## ]]
+ What is 1+1?
+
+ [[ ## final_generated_code ## ]]
+ result = 1+1
+
+ [[ ## code_output ## ]]
+ 2
+
+ Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
),
},
]
diff --git a/tests/predict/test_react.py b/tests/predict/test_react.py
index 9b187471b..8ee626aeb 100644
--- a/tests/predict/test_react.py
+++ b/tests/predict/test_react.py
@@ -8,8 +8,7 @@ def test_example_no_tools():
# Createa a simple dataset which the model will use with the Retrieve tool.
lm = dspy.utils.DummyLM(
[
- "Initial thoughts", # Thought_1
- "Finish[blue]", # Action_1
+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nFinish[blue]",
]
)
dspy.settings.configure(lm=lm, rm=dummy_rm())
@@ -24,28 +23,13 @@ def test_example_no_tools():
result = program(question=question)
assert result.answer == "blue"
- # For debugging
- print("---")
- for row in lm.history:
- print(row["prompt"])
- print("Response:", row["response"]["choices"][0]["text"])
- print("---")
-
- assert lm.get_convo(-1).endswith(
- "Question: What is the color of the sky?\n"
- "Thought 1: Initial thoughts\n"
- "Action 1: Finish[blue]"
- )
-
def test_example_search():
# Createa a simple dataset which the model will use with the Retrieve tool.
lm = dspy.utils.DummyLM(
[
- "Initial thoughts", # Thought_1
- "Search[the color of the sky]", # Thought_1
- "More thoughts", # Thought_2
- "Finish[blue]", # Action_2
+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nSearch[the color of the sky]",
+ "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nFinish[blue]\n\n",
]
)
rm = dummy_rm(
@@ -72,21 +56,6 @@ def test_example_search():
result = program(question=question)
assert result.answer == "blue"
- # For debugging
- print(lm.get_convo(-1))
-
- assert lm.get_convo(-1).endswith(
- "Question: What is the color of the sky?\n\n"
- "Thought 1: Initial thoughts\n\n"
- "Action 1: Search[the color of the sky]\n\n"
- "Observation 1:\n"
- "[1] «We all know the color of the sky is blue.»\n"
- "[2] «Somethng about the sky colors»\n"
- "[3] «This sentence is completely irellevant to answer the question.»\n\n"
- "Thought 2: More thoughts\n\n"
- "Action 2: Finish[blue]"
- )
-
class DummyTool1:
name = "Tool1"
@@ -122,12 +91,9 @@ class DummyTool2:
def test_custom_tools():
lm = dspy.utils.DummyLM(
[
- "Initial thoughts",
- "Tool1[foo]",
- "More thoughts",
- "Tool2[bar]",
- "Even more thoughts",
- "Finish[baz]",
+ "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nTool1[foo]",
+ "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nTool2[bar]",
+ "[[ ## Thought_3 ## ]]\nEven more thoughts\n\n[[ ## Action_3 ## ]]\nFinish[baz]",
]
)
dspy.settings.configure(lm=lm)
@@ -143,17 +109,6 @@ def test_custom_tools():
# each tool should be called only once
assert tool1.num_calls == 1
assert tool2.num_calls == 1
- assert lm.get_convo(-1).endswith(
- "Question: What is the color of the sky?\n\n"
- "Thought 1: Initial thoughts\n\n"
- "Action 1: Tool1[foo]\n\n"
- "Observation 1: tool 1 output\n\n"
- "Thought 2: More thoughts\n\n"
- "Action 2: Tool2[bar]\n\n"
- "Observation 2: tool 2 output\n\n"
- "Thought 3: Even more thoughts\n\n"
- "Action 3: Finish[baz]"
- )
def test_signature_instructions():
diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.py
index 5ceee8a62..16050e355 100644
--- a/tests/predict/test_retry.py
+++ b/tests/predict/test_retry.py
@@ -14,7 +14,7 @@ def test_retry_simple():
assert f"past_{field}" in retry_module.new_signature.input_fields
assert "feedback" in retry_module.new_signature.input_fields
- lm = DummyLM(["blue"])
+ lm = DummyLM(["[[ ## answer ## ]]\nblue"])
dspy.settings.configure(lm=lm)
result = retry_module.forward(
question="What color is the sky?",
@@ -23,18 +23,10 @@ def test_retry_simple():
)
assert result.answer == "blue"
- print(lm.get_convo(-1))
- assert lm.get_convo(-1).endswith(
- "Question: What color is the sky?\n\n"
- "Previous Answer: red\n\n"
- "Instructions: Try harder\n\n"
- "Answer: blue"
- )
-
def test_retry_forward_with_feedback():
# First we make a mistake, then we fix it
- lm = DummyLM(["red", "blue"])
+ lm = DummyLM(["[[ ## answer ## ]]\nred", "[[ ## answer ## ]]\nblue"])
dspy.settings.configure(lm=lm, trace=[])
class SimpleModule(dspy.Module):
@@ -58,18 +50,10 @@ def test_retry_forward_with_feedback():
assert result.answer == "blue"
- print(lm.get_convo(-1))
- assert lm.get_convo(-1).endswith(
- "Question: What color is the sky?\n\n"
- "Previous Answer: red\n\n"
- "Instructions: Please think harder\n\n"
- "Answer: blue"
- )
-
def test_retry_forward_with_typed_predictor():
# First we make a mistake, then we fix it
- lm = DummyLM(['{"answer":"red"}', '{"answer":"blue"}'])
+ lm = DummyLM(['[[ ## output ## ]]\n{"answer":"red"}', '[[ ## output ## ]]\n{"answer":"blue"}'])
dspy.settings.configure(lm=lm, trace=[])
class AnswerQuestion(dspy.Signature):
@@ -103,9 +87,3 @@ def test_retry_forward_with_typed_predictor():
result = program(question="What color is the sky?")
assert result.answer == "blue"
- assert lm.get_convo(-1).endswith(
- 'Input: {"question":"What color is the sky?"}\n\n'
- 'Previous Output: {"answer":"red"}\n\n'
- 'Instructions: Please think harder\n\n'
- 'Output: {"answer":"blue"}'
- )
diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.py
index 52f6f7477..292bafdf2 100644
--- a/tests/signatures/test_signature.py
+++ b/tests/signatures/test_signature.py
@@ -182,6 +182,9 @@ def test_insantiating2():
def test_multiline_instructions():
+ lm = DummyLM(["[[ ## output ## ]]\nshort answer"])
+ dspy.settings.configure(lm=lm)
+
class MySignature(Signature):
"""First line
Second line
@@ -190,15 +193,8 @@ def test_multiline_instructions():
output = OutputField()
predictor = dspy.Predict(MySignature)
-
- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])
- dspy.settings.configure(lm=lm)
assert predictor().output == "short answer"
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
assert lm.get_convo(-1)[0] == [
{
"role": "system",
@@ -219,7 +215,7 @@ def test_multiline_instructions():
[[ ## completed ## ]]
- In adhering to this structure, your objective is:
+ In adhering to this structure, your objective is:
First line
Second line
Third line"""
diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.py
index 05431a632..ed3e61446 100644
--- a/tests/teleprompt/test_bootstrap.py
+++ b/tests/teleprompt/test_bootstrap.py
@@ -82,7 +82,7 @@ def test_bootstrap_effectiveness():
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
@@ -105,7 +105,7 @@ def test_bootstrap_effectiveness():
[[ ## completed ## ]]
- In adhering to this structure, your objective is:
+ In adhering to this structure, your objective is:
Given the fields `input`, produce the fields `output`."""
),
},
diff --git a/tests/teleprompt/test_copro_optimizer.py b/tests/teleprompt/test_copro_optimizer.py
index 2618367ca..7f391f351 100644
--- a/tests/teleprompt/test_copro_optimizer.py
+++ b/tests/teleprompt/test_copro_optimizer.py
@@ -123,7 +123,7 @@ def test_optimization_and_output_verification():
for message in lm.get_convo(-1)[0]:
print("----")
- print(message["content"])
+ print(textwrap.indent(message["content"], " "))
print("----")
assert lm.get_convo(-1)[0] == [
{
@@ -150,7 +150,7 @@ def test_optimization_and_output_verification():
[[ ## completed ## ]]
- In adhering to this structure, your objective is:
+ In adhering to this structure, your objective is:
Optimized Prompt"""
),
},
diff --git a/tests/teleprompt/test_mipro_optimizer.py b/tests/teleprompt/test_mipro_optimizer.py
deleted file mode 100644
index b586068f9..000000000
--- a/tests/teleprompt/test_mipro_optimizer.py
+++ /dev/null
@@ -1,246 +0,0 @@
-import re
-import textwrap
-
-import pytest
-
-import dspy
-from dsp.modules import LM
-from dspy import Example
-from dspy.teleprompt.signature_opt_bayesian import MIPRO
-from dspy.utils.dummies import DummyLM
-
-
-# Define a simple metric function for testing
-def simple_metric(example, prediction, trace=None):
- # Simplified metric for testing: true if prediction matches expected output
- return example.output == prediction.output
-
-
-# Some example data
-capitals = {
- "Germany": "Berlin",
- "France": "Paris",
- "Denmark": "Copenhagen",
- "Sweden": "Stockholm",
- "Norway": "Oslo",
-}
-# Not used for training data
-extra_capitals = {
- "Spain": "Madrid",
- "Portugal": "Lisbon",
- "Italy": "Rome",
-}
-
-# Example training and validation sets
-trainset = [
- Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
- Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),
-] + [
- Example(input=f"What is the capital of {country}?", output=capital).with_inputs("input")
- for country, capital in capitals.items()
-]
-
-
-class ConditionalLM(LM):
- def __init__(self):
- super().__init__("conditional-lm")
-
- def basic_request(self, prompt, num_candidates=1, **kwargs):
- # If we are in the "optimization" stage, we don't say much.
- if prompt.endswith("Observations:"):
- answer = " (*silence*)"
- elif prompt.endswith("Proposed Instruction:"):
- answer = " Input: "
- elif prompt.endswith("Proposed Prefix For Output Field:"):
- answer = " Output: "
- elif prompt.endswith("Summary:"):
- answer = " summarizing..."
- else:
- pairs = re.findall(r"Input: (.*?)\n(?:Reasoning:.*?\n)?Output: (.*?)\n", prompt, re.DOTALL)
-
- # breakpoint()
- print("PROMPT:", prompt)
- print("PAIRS:", pairs)
-
- last = re.search(r"Input: (.*)\nReasoning: (.*)$", prompt)
- current_question = last.group(1)
-
- if match := re.match(r"What is the capital of (.*?)\?", current_question):
- country = match.group(1)
- # If we had a previous example of a question about a capital, the model
- # has learned the format, and will answer with question correctly.
- if any("capital" in question for question, _ in pairs):
- answer = (capitals | extra_capitals)[country]
- # Otherwise, it is confused and will answer with the country's name.
- else:
- answer = country
- # For other questions, the model will answer with the last word of the question.
- else:
- answer = current_question.split()[-1]
-
- answer = "think deeply.\nOutput: " + answer
-
- RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m"
- print("=== DummyLM ===")
- print(prompt, end="")
- print(f"{RED}{answer}{RESET}")
- print("===")
-
- dummy_response = {"choices": []}
- for _ in range(num_candidates):
- dummy_response["choices"].append(
- {
- "text": answer,
- "finish_reason": "done",
- }
- )
-
- # Simulate processing and storing the request and response.
- history_entry = {
- "prompt": prompt,
- "response": dummy_response,
- "kwargs": kwargs,
- "raw_kwargs": kwargs,
- }
- self.history.append(history_entry)
-
- return dummy_response
-
- def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
- response = self.basic_request(prompt, **kwargs)
- return [choice["text"] for choice in response["choices"]]
-
- def get_convo(self, index):
- """get the prompt + anwer from the ith message"""
- return self.history[index]["prompt"] + " " + self.history[index]["response"]["choices"][0]["text"]
-
-
-def test_bayesian_signature_optimizer_initialization():
- optimizer = MIPRO(metric=simple_metric, num_candidates=10, init_temperature=1.4, verbose=True, track_stats=True)
- assert optimizer.metric == simple_metric, "Metric not correctly initialized"
- assert optimizer.num_candidates == 10, "Incorrect 'num_candidates' parameter initialization"
- assert optimizer.init_temperature == 1.4, "Initial temperature not correctly initialized"
- assert optimizer.verbose is True, "Verbose flag not correctly initialized"
- assert optimizer.track_stats is True, "Track stats flag not correctly initialized"
-
-
-class SimpleModule(dspy.Module):
- def __init__(self, signature):
- super().__init__()
- # SignatureOptimizer doesn't work with dspy.Predict
- self.predictor = dspy.ChainOfThought(signature)
-
- def forward(self, **kwargs):
- return self.predictor(**kwargs)
-
-
-def test_signature_optimizer_optimization_process():
- lm = ConditionalLM()
- dspy.settings.configure(lm=lm)
-
- student = SimpleModule(signature="input -> output")
-
- optimizer = MIPRO(
- metric=simple_metric,
- num_candidates=10,
- init_temperature=1.4,
- verbose=False,
- track_stats=False,
- )
-
- # Adjustments: Include required parameters for the compile method
- optimized_student = optimizer.compile(
- student=student,
- trainset=trainset,
- num_trials=10,
- max_bootstrapped_demos=3,
- max_labeled_demos=5,
- eval_kwargs={"num_threads": 1, "display_progress": False},
- requires_permission_to_run=False,
- )
-
- assert len(optimized_student.predictor.demos) == 5
-
-
-def test_signature_optimizer_bad_lm():
- dspy.settings.configure(lm=DummyLM([f"Optimized instruction {i}" for i in range(30)]))
- student = SimpleModule(signature="input -> output")
- optimizer = MIPRO(
- metric=simple_metric,
- num_candidates=10,
- init_temperature=1.4,
- verbose=False,
- track_stats=False,
- )
-
- # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DummyLM,
- # which only outputs "Optimized instruction i" this means that none of the bootstrapped examples are successful,
- # and therefore the set of examples that we're using to generate new prompts is empty
- with pytest.raises(ValueError):
- _optimized_student = optimizer.compile(
- student=student,
- trainset=trainset,
- num_trials=10,
- max_bootstrapped_demos=3,
- max_labeled_demos=5,
- eval_kwargs={"num_threads": 1, "display_progress": False},
- requires_permission_to_run=False,
- )
-
-
-def test_optimization_and_output_verification():
- # Make a language model that is always right, except on the last
- # example in the train set.
- lm = ConditionalLM()
- dspy.settings.configure(lm=lm)
-
- optimizer = MIPRO(
- metric=simple_metric,
- num_candidates=10,
- init_temperature=1.4,
- verbose=False,
- track_stats=True,
- )
-
- student = SimpleModule("input -> output")
-
- # Compile the student with the optimizer
- optimized_student = optimizer.compile(
- student=student,
- trainset=trainset,
- num_trials=4,
- max_bootstrapped_demos=2,
- max_labeled_demos=3,
- eval_kwargs={"num_threads": 1, "display_progress": False},
- requires_permission_to_run=False,
- )
-
- # Simulate calling the optimized student with a new input
- test_input = "What is the capital of Spain?"
- prediction = optimized_student(input=test_input)
-
- print("CORRECT ANSWER")
- print(lm.get_convo(-1))
-
- assert prediction.output == "Madrid"
-
- for message in lm.get_convo(-1)[0]:
- print("----")
- print(message["content"])
- print("----")
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- """
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- """
- ),
- },
- ]