Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): update DummyLM to use ChatAdapter's format_field to format outputs

Michael Jones738ea87a348a
diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py
index 3288facf5..bdc51f0aa 100644
--- a/dspy/utils/dummies.py
+++ b/dspy/utils/dummies.py
@@ -7,7 +7,7 @@ import numpy as np
from dsp.modules import LM as DSPLM
from dsp.utils.utils import dotdict
-from dspy.adapters.chat_adapter import field_header_pattern
+from dspy.adapters.chat_adapter import field_header_pattern, format_fields
from dspy.clients.lm import LM
@@ -98,7 +98,7 @@ class DSPDummyLM(DSPLM):
class DummyLM(LM):
- def __init__(self, answers: Union[list[str], dict[str, str]], follow_examples: bool = False):
+ def __init__(self, answers: Union[list[dict[str, str]], dict[str, dict[str, str]]], follow_examples: bool = False):
super().__init__("dummy", "chat", 0.0, 1000, True)
self.answers = answers
if isinstance(answers, list):
@@ -133,10 +133,13 @@ class DummyLM(LM):
outputs.append(self._use_example(messages))
elif isinstance(self.answers, dict):
outputs.append(
- next((v for k, v in self.answers.items() if k in messages[-1]["content"]), "No more responses")
+ next(
+ (format_fields(v) for k, v in self.answers.items() if k in messages[-1]["content"]),
+ "No more responses",
+ )
)
else:
- outputs.append(next(self.answers, "No more responses"))
+ outputs.append(format_fields(next(self.answers, {"answer": "No more responses"})))
# Logging, with removed api key & where `cost` is None on cache hit.
kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.py
index 01c3e5739..a8552c7aa 100644
--- a/tests/evaluate/test_evaluate.py
+++ b/tests/evaluate/test_evaluate.py
@@ -37,8 +37,8 @@ def test_evaluate_call():
dspy.settings.configure(
lm=DummyLM(
{
- "What is 1+1?": "[[ ## answer ## ]]\n2",
- "What is 2+2?": "[[ ## answer ## ]]\n4",
+ "What is 1+1?": {"answer": "2"},
+ "What is 2+2?": {"answer": "4"},
}
)
)
@@ -55,9 +55,7 @@ def test_evaluate_call():
def test_multithread_evaluate_call():
- dspy.settings.configure(
- lm=DummyLM({"What is 1+1?": "[[ ## answer ## ]]\n2", "What is 2+2?": "[[ ## answer ## ]]\n4"})
- )
+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
assert program(question="What is 1+1?").answer == "2"
@@ -80,9 +78,7 @@ def test_multi_thread_evaluate_call_cancelled(monkeypatch):
time.sleep(1)
return super().__call__(*args, **kwargs)
- dspy.settings.configure(
- lm=SlowLM({"What is 1+1?": "[[ ## answer ## ]]\n2", "What is 2+2?": "[[ ## answer ## ]]\n4"})
- )
+ dspy.settings.configure(lm=SlowLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
@@ -112,9 +108,7 @@ def test_multi_thread_evaluate_call_cancelled(monkeypatch):
def test_evaluate_call_bad():
- dspy.settings.configure(
- lm=DummyLM({"What is 1+1?": "[[ ## answer ## ]]\n0", "What is 2+2?": "[[ ## answer ## ]]\n0"})
- )
+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": {"answer": "0"}, "What is 2+2?": {"answer": "0"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
ev = Evaluate(
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 1f0242763..61ac8c15a 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -21,7 +21,7 @@ def test_simple():
"""Think of a hard factual question about a topic."""
expected = "What is the speed of light?"
- lm = DummyLM([f"[[ ## hard_question ## ]]\n{expected}"])
+ lm = DummyLM([{"hard_question": expected}])
dspy.settings.configure(lm=lm)
question = hard_question(topic="Physics")
@@ -36,7 +36,7 @@ def test_list_output():
pass
expected = ["What is the speed of light?", "What is the speed of sound?"]
- lm = DummyLM(['[[ ## hard_questions ## ]]\n["What is the speed of light?", "What is the speed of sound?"]'])
+ lm = DummyLM([{"hard_questions": '["What is the speed of light?", "What is the speed of sound?"]'}])
dspy.settings.configure(lm=lm)
question = hard_questions(topics=["Physics", "Music"])
@@ -54,7 +54,7 @@ def test_simple_type():
"""Think of a hard factual question about a topic."""
expected = "What is the speed of light?"
- lm = DummyLM([f'[[ ## hard_question ## ]]\n{{"value": "{expected}"}}'])
+ lm = DummyLM([{"hard_question": f'{{"value": "{expected}"}}'}])
dspy.settings.configure(lm=lm)
question = hard_question(topic="Physics")
@@ -75,7 +75,7 @@ def test_simple_type_input():
pass
question = Question(value="What is the speed of light?")
- lm = DummyLM([f'[[ ## answer ## ]]\n{{"value": "3e8"}}'])
+ lm = DummyLM([{"answer": '{"value": "3e8"}'}])
dspy.settings.configure(lm=lm)
result = answer(question=question)
@@ -110,10 +110,10 @@ def test_simple_class():
lm = DummyLM(
[
- "[[ ## hard_question ## ]]\nWhat is the speed of light?",
- "[[ ## reasoning ## ]]\nSome bad reasoning, 3e8 m/s.\n\n[[ ## answer ## ]]\n3e8", # Bad answer 1
- "[[ ## json_object ## ]]\n{...}", # Model is asked to create an example
- f"[[ ## reasoning ## ]]\nSome good reasoning, 3e8 m/s.\n\n[[ ## answer ## ]]\n{expected.model_dump_json()}", # Good answer
+ {"hard_question": "What is the speed of light?"},
+ {"reasoning": "Some bad reasoning, 3e8 m/s.", "answer": "3e8"}, # Bad answer 1
+ {"json_object": "{...}"}, # Model is asked to create an example
+ {"reasoning": "Some good reasoning, 3e8 m/s.", "answer": f"{expected.model_dump_json()}"}, # Good answer
]
)
dspy.settings.configure(lm=lm)
@@ -143,7 +143,7 @@ def test_simple_oop():
expected = "What is the speed of light?"
lm = DummyLM(
[
- f"[[ ## output ## ]]\n{Question(value=expected).model_dump_json()}",
+ {"output": f"{Question(value=expected).model_dump_json()}"},
]
)
dspy.settings.configure(lm=lm)
@@ -242,24 +242,6 @@ def test_bootstrap_effectiveness():
prediction = compiled_student(input=trainset[0].input)
assert prediction == trainset[0].output
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": "Your input fields are:\n1. `input` (str)\n\nYour output fields are:\n1. `output` (str)\n\nAll interactions will be structured in the following way, with the appropriate values filled in.\n\n[[ ## input ## ]]\n{input}\n\n[[ ## output ## ]]\n{output}\n\n[[ ## completed ## ]]\n\nIn adhering to this structure, your objective is: \n Given the fields `input`, produce the fields `output`.",
- },
- {
- "role": "user",
- "content": "[[ ## input ## ]]\nWhat is the color of the sky?\n\nRespond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",
- },
- {"role": "assistant", "content": "[[ ## output ## ]]\nblue\n\n[[ ## completed ## ]]"},
- {
- "role": "user",
- "content": "[[ ## input ## ]]\nWhat is the color of the sky?\n\nRespond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",
- },
- ]
-
- assert lm.get_convo(-1)[1] == ["[[ ## output ## ]]\nblue\n\n[[ ## completed ## ]]"]
-
def test_regex():
class TravelInformation(BaseModel):
@@ -281,11 +263,11 @@ def test_regex():
lm = DummyLM(
[
# Example with a bad origin code.
- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},
# Example to help the model understand
- "[[ ## json_object ## ]]\n{...}",
+ {"json_object": "{...}"},
# Fixed
- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}'},
]
)
dspy.settings.configure(lm=lm)
@@ -341,11 +323,12 @@ def test_custom_model_validate_json():
[
# Example with a bad origin code.
(
- "[[ ## flight_information ## ]]\nHere is your json: "
- "{"
- '"origin": {"code":"JFK", "lat":40.6446, "lon":-73.7797}, '
- '"destination": {"code":"LAX", "lat":33.942791, "lon":-118.410042}, '
- '"date": "2022-12-25"}'
+ {
+ "flight_information": "Here is your json: {"
+ '"origin": {"code":"JFK", "lat":40.6446, "lon":-73.7797}, '
+ '"destination": {"code":"LAX", "lat":33.942791, "lon":-118.410042}, '
+ '"date": "2022-12-25"}'
+ }
),
]
)
@@ -370,8 +353,8 @@ def test_raises():
lm = DummyLM(
[
- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "bad date"}',
+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},
+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "bad date"}'},
]
)
dspy.settings.configure(lm=lm)
@@ -393,11 +376,11 @@ def test_multi_errors():
lm = DummyLM(
[
# First origin is wrong, then destination, then all is good
- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
- "[[ ## json_object ## ]]\n{...}", # Example to help the model understand
- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}',
- "[[ ## json_object ## ]]\n{...}", # Example to help the model understand
- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},
+ {"json_object": "{...}"}, # Example to help the model understand
+ {"flight_information": '{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}'},
+ {"json_object": "{...}"}, # Example to help the model understand
+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}'},
]
)
dspy.settings.configure(lm=lm)
@@ -406,57 +389,6 @@ def test_multi_errors():
origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25)
)
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `email` (str)
- 2. `error_flight_information_0` (str): An error to avoid in the future
- 3. `error_flight_information_1` (str): An error to avoid in the future
-
- Your output fields are:
- 1. `flight_information` (TravelInformation): ${flight_information}. Respond with a single JSON object. JSON Schema: {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## email ## ]]
- {email}
-
- [[ ## error_flight_information_0 ## ]]
- {error_flight_information_0}
-
- [[ ## error_flight_information_1 ## ]]
- {error_flight_information_1}
-
- [[ ## flight_information ## ]]
- {flight_information}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `email`, produce the fields `flight_information`."""
- ), # noqa
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## email ## ]]
- Some email
-
- [[ ## error_flight_information_0 ## ]]
- String should match pattern '^[A-Z]{3}$': origin (error type: string_pattern_mismatch)
-
- [[ ## error_flight_information_1 ## ]]
- String should match pattern '^[A-Z]{3}$': destination (error type: string_pattern_mismatch)
-
- Respond with the corresponding output fields, starting with the field `flight_information`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_field_validator():
class UserDetails(BaseModel):
@@ -476,54 +408,12 @@ def test_field_validator():
# Keep making the mistake (lower case name) until we run
# out of retries.
- lm = DummyLM(
- [
- '[[ ## get_user_details ## ]]\n{"name": "lower case name", "age": 25}',
- ]
- * 10
- )
+ lm = DummyLM([{"get_user_details": '{"name": "lower case name", "age": 25}'}] * 10)
dspy.settings.configure(lm=lm)
with pytest.raises(ValueError):
get_user_details()
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `json_schema` (str)
-
- Your output fields are:
- 1. `json_object` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## json_schema ## ]]
- {json_schema}
-
- [[ ## json_object ## ]]
- {json_object}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Make a very succinct json object that validates with the following schema"""
- ), # noqa
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## json_schema ## ]]
- {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}
-
- Respond with the corresponding output fields, starting with the field `json_object`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_annotated_field():
@predictor
@@ -531,7 +421,7 @@ def test_annotated_field():
pass
# First try 0, which fails, then try 0.5, which passes
- lm = DummyLM(["[[ ## test ## ]]\n0", "[[ ## test ## ]]\n0.5"])
+ lm = DummyLM([{"test": "0"}, {"test": "0.5"}])
dspy.settings.configure(lm=lm)
output = test(input="input")
@@ -540,7 +430,7 @@ def test_annotated_field():
def test_multiple_outputs():
- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])
+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])
dspy.settings.configure(lm=lm)
test = TypedPredictor("input -> output")
@@ -549,7 +439,7 @@ def test_multiple_outputs():
def test_multiple_outputs_int():
- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])
+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])
dspy.settings.configure(lm=lm)
class TestSignature(dspy.Signature):
@@ -566,9 +456,9 @@ def test_multiple_outputs_int_cot():
# Note: Multiple outputs only work when the language model "speculatively" generates all the outputs in one go.
lm = DummyLM(
[
- "[[ ## reasoning ## ]]\nthoughts 0\n\n[[ ## output ## ]]\n0\n",
- "[[ ## reasoning ## ]]\nthoughts 1\n\n[[ ## output ## ]]\n1\n",
- "[[ ## reasoning ## ]]\nthoughts 2\n\n[[ ## output ## ]]\n2\n",
+ {"reasoning": "thoughts 0", "output": "0"},
+ {"reasoning": "thoughts 1", "output": "1"},
+ {"reasoning": "thoughts 2", "output": "2"},
]
)
dspy.settings.configure(lm=lm)
@@ -580,7 +470,7 @@ def test_multiple_outputs_int_cot():
def test_parse_type_string():
- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])
+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])
dspy.settings.configure(lm=lm)
test = TypedPredictor("input:int -> output:int")
@@ -590,7 +480,7 @@ def test_parse_type_string():
def test_literal():
- lm = DummyLM(['[[ ## f ## ]]\n"2"', '[[ ## f ## ]]\n"3"'])
+ lm = DummyLM([{"f": '"2"'}, {"f": '"3"'}])
dspy.settings.configure(lm=lm)
@predictor
@@ -601,7 +491,7 @@ def test_literal():
def test_literal_missmatch():
- lm = DummyLM([f'[[ ## f ## ]]\n"{i}"' for i in range(5, 100)])
+ lm = DummyLM([{"f": f"{i}"} for i in range(5, 100)])
dspy.settings.configure(lm=lm)
@predictor(max_retries=1)
@@ -615,7 +505,7 @@ def test_literal_missmatch():
def test_literal_int():
- lm = DummyLM(["[[ ## f ## ]]\n2", "[[ ## f ## ]]\n3"])
+ lm = DummyLM([{"f": "2"}, {"f": "3"}])
dspy.settings.configure(lm=lm)
@predictor
@@ -626,7 +516,7 @@ def test_literal_int():
def test_literal_int_missmatch():
- lm = DummyLM([f"[[ ## f ## ]]\n{i}" for i in range(5, 100)])
+ lm = DummyLM([{"f": f"{i}"} for i in range(5, 100)])
dspy.settings.configure(lm=lm)
@predictor(max_retries=1)
@@ -645,8 +535,8 @@ def test_fields_on_base_signature():
lm = DummyLM(
[
- "[[ ## output ## ]]\n2.1", # Bad output
- "[[ ## output ## ]]\n0.5", # Good output
+ {"output": "2.1"}, # Bad output
+ {"output": "0.5"}, # Good output
]
)
dspy.settings.configure(lm=lm)
@@ -668,12 +558,12 @@ def test_synthetic_data_gen():
lm = DummyLM(
[
- '[[ ## fact ## ]]\n{"fact": "The sky is blue", "varacity": true}',
- '[[ ## fact ## ]]\n{"fact": "The sky is green", "varacity": false}',
- '[[ ## fact ## ]]\n{"fact": "The sky is red", "varacity": true}',
- '[[ ## fact ## ]]\n{"fact": "The earth is flat", "varacity": false}',
- '[[ ## fact ## ]]\n{"fact": "The earth is round", "varacity": true}',
- '[[ ## fact ## ]]\n{"fact": "The earth is a cube", "varacity": false}',
+ {"fact": '{"fact": "The sky is blue", "varacity": true}'},
+ {"fact": '{"fact": "The sky is green", "varacity": false}'},
+ {"fact": '{"fact": "The sky is red", "varacity": true}'},
+ {"fact": '{"fact": "The earth is flat", "varacity": false}'},
+ {"fact": '{"fact": "The earth is round", "varacity": true}'},
+ {"fact": '{"fact": "The earth is a cube", "varacity": false}'},
]
)
dspy.settings.configure(lm=lm)
@@ -709,7 +599,7 @@ def test_list_input2():
program = TypedChainOfThought(ScoredSignature)
- lm = DummyLM(["[[ ## reasoning ## ]]\nThoughts\n\n[[ ## proposed_signature ## ]]\nOutput"])
+ lm = DummyLM([{"reasoning": "Thoughts", "proposed_signature": "Output"}])
dspy.settings.configure(lm=lm)
output = program(
@@ -724,49 +614,6 @@ def test_list_input2():
assert output == "Output"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `attempted_signatures` (list[ScoredString])
-
- Your output fields are:
- 1. `reasoning` (str): ${produce the proposed_signature}. We ...
- 2. `proposed_signature` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## attempted_signatures ## ]]
- {attempted_signatures}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## proposed_signature ## ]]
- {proposed_signature}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `attempted_signatures`, produce the fields `proposed_signature`."""
- ), # noqa
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## attempted_signatures ## ]]
- [1] «string='string 1' score=0.5»
- [2] «string='string 2' score=0.4»
- [3] «string='string 3' score=0.3»
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `proposed_signature`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_custom_reasoning_field():
class Question(pydantic.BaseModel):
@@ -784,7 +631,7 @@ def test_custom_reasoning_field():
program = TypedChainOfThought(QuestionSignature, reasoning=reasoning)
expected = "What is the speed of light?"
- lm = DummyLM([f'[[ ## reasoning ## ]]\nThoughts\n\n[[ ## question ## ]]\n{{"value": "{expected}"}}'])
+ lm = DummyLM([{"reasoning": "Thoughts", "question": f'{{"value": "{expected}"}}'}])
dspy.settings.configure(lm=lm)
output = program(topic="Physics")
@@ -792,47 +639,6 @@ def test_custom_reasoning_field():
assert isinstance(output.question, Question)
assert output.question.value == expected
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `topic` (str)
-
- Your output fields are:
- 1. `reasoning` (str): ${topic}, we should ...
- 2. `question` (Question): ${question}. Respond with a single JSON object. JSON Schema: {"properties": {"value": {"title": "Value", "type": "string"}}, "required": ["value"], "title": "Question", "type": "object"}
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## topic ## ]]
- {topic}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## question ## ]]
- {question}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `topic`, produce the fields `question`."""
- ), # noqa
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## topic ## ]]
- Physics
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `question`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_generic_signature():
T = TypeVar("T")
@@ -845,7 +651,7 @@ def test_generic_signature():
predictor = TypedPredictor(GenericSignature[int])
assert predictor.signature.instructions == "My signature"
- lm = DummyLM(["[[ ## output ## ]]\n23"])
+ lm = DummyLM([{"output": "23"}])
dspy.settings.configure(lm=lm)
assert predictor().output == 23
@@ -858,7 +664,7 @@ def test_field_validator_in_signature():
@pydantic.field_validator("a")
@classmethod
def space_in_a(cls, a: str) -> str:
- if not " " in a:
+ if " " not in a:
raise ValueError("a must contain a space")
return a
@@ -883,10 +689,10 @@ def test_lm_as_validator():
lm = DummyLM(
[
- "[[ ## next_square ## ]]\n3",
- "[[ ## is_square ## ]]\nFalse",
- "[[ ## next_square ## ]]\n4",
- "[[ ## is_square ## ]]\nTrue",
+ {"next_square": "3"},
+ {"is_square": "False"},
+ {"next_square": "4"},
+ {"is_square": "True"},
]
)
dspy.settings.configure(lm=lm)
@@ -909,7 +715,7 @@ def test_annotated_validator():
n: int = dspy.InputField()
next_square: Annotated[int, AfterValidator(is_square)] = dspy.OutputField()
- lm = DummyLM(["[[ ## next_square ## ]]\n3", "[[ ## next_square ## ]]\n4"])
+ lm = DummyLM([{"next_square": "3"}, {"next_square": "4"}])
dspy.settings.configure(lm=lm)
m = TypedPredictor(MySignature)(n=2).next_square
@@ -928,7 +734,7 @@ def test_annotated_validator_functional():
def next_square(n: int) -> Annotated[int, AfterValidator(is_square)]:
"""What is the next square number after n?"""
- lm = DummyLM(["[[ ## next_square ## ]]\n3", "[[ ## next_square ## ]]\n4"])
+ lm = DummyLM([{"next_square": "3"}, {"next_square": "4"}])
dspy.settings.configure(lm=lm)
m = next_square(n=2)
@@ -946,68 +752,11 @@ def test_demos():
trainset=[ex.with_inputs("input") for ex in demos],
)
- lm = DummyLM(["[[ ## output ## ]]\nParis"])
+ lm = DummyLM([{"output": "Paris"}])
dspy.settings.configure(lm=lm)
assert program(input="What is the capital of France?").output == "Paris"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `input` (str)
-
- Your output fields are:
- 1. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## input ## ]]
- {input}
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `input`, produce the fields `output`."""
- ), # noqa
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- What is the speed of light?
-
- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
- ),
- },
- {
- "role": "assistant",
- "content": textwrap.dedent(
- """\
- [[ ## output ## ]]
- 3e8
-
- [[ ## completed ## ]]"""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- What is the capital of France?
-
- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_demos_missing_input_in_demo():
demos = [dspy.Example(input="What is the speed of light?", output="3e8")]
@@ -1015,7 +764,7 @@ def test_demos_missing_input_in_demo():
student=dspy.TypedPredictor("input -> output, thoughts"),
trainset=[ex.with_inputs("input") for ex in demos],
)
- lm = DummyLM(["[[ ## thoughts ## ]]\nMy thoughts\n\n[[ ## output ## ]]\nParis"])
+ lm = DummyLM([{"thoughts": "My thoughts", "output": "Paris"}])
dspy.settings.configure(lm=lm)
assert program(input="What is the capital of France?").output == "Paris"
@@ -1024,10 +773,10 @@ def test_conlist():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## make_numbers ## ]]\n[]",
- "[[ ## make_numbers ## ]]\n[1]",
- "[[ ## make_numbers ## ]]\n[1, 2]",
- "[[ ## make_numbers ## ]]\n[1, 2, 3]",
+ {"make_numbers": "[]"},
+ {"make_numbers": "[1]"},
+ {"make_numbers": "[1, 2]"},
+ {"make_numbers": "[1, 2, 3]"},
]
)
)
@@ -1043,10 +792,10 @@ def test_conlist2():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## output ## ]]\n[]",
- "[[ ## output ## ]]\n[1]",
- "[[ ## output ## ]]\n[1, 2]",
- "[[ ## output ## ]]\n[1, 2, 3]",
+ {"output": "[]"},
+ {"output": "[1]"},
+ {"output": "[1, 2]"},
+ {"output": "[1, 2, 3]"},
]
)
)
@@ -1067,7 +816,7 @@ def test_model_validator():
raise ValueError(f"category not in {self.allowed_categories}")
return self
- lm = DummyLM(["[[ ## category ## ]]\nhorse", "[[ ## category ## ]]\ndog"])
+ lm = DummyLM([{"category": "horse"}, {"category": "dog"}])
dspy.settings.configure(lm=lm)
predictor = TypedPredictor(MySignature)
diff --git a/tests/functional/test_signature_opt_typed.py b/tests/functional/test_signature_opt_typed.py
index 125cec30b..6778f6b69 100644
--- a/tests/functional/test_signature_opt_typed.py
+++ b/tests/functional/test_signature_opt_typed.py
@@ -1,7 +1,5 @@
import json
-from typing import Generic, TypeVar
-import pydantic
from pydantic_core import to_jsonable_python
import dspy
@@ -103,12 +101,14 @@ def test_opt():
question: str = dspy.InputField()
answer: str = dspy.OutputField()
- qa_model = DummyLM(["[[ ## answer ## ]]\nfoo"] * 100)
+ qa_model = DummyLM([{"answer": "foo"}] * 100)
prompt_model = DummyLM(
[
+ {
+ "reasoning": "some thoughts",
+ "proposed_signatures": '[{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]',
+ }
# Seed prompts
- "[[ ## reasoning ## ]]\nsome thoughts\n\n"
- '[[ ## proposed_signatures ## ]]\n[{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]',
]
)
dspy.settings.configure(lm=qa_model)
@@ -165,8 +165,8 @@ def test_opt_composed():
qa_model = DummyLM([])
prompt_model = DummyLM(
[
- f"[[ ## reasoning ## ]]\nsome thoughts\n\n[[ ## proposed_signatures ## ]]\n{json.dumps([to_jsonable_python(info1)])}",
- f"[[ ## reasoning ## ]]\nsome thoughts\n\n[[ ## proposed_signatures ## ]]\n{json.dumps([to_jsonable_python(info2)])}",
+ {"reasoning": "some thoughts", "proposed_signatures": json.dumps([to_jsonable_python(info1)])},
+ {"reasoning": "some thoughts", "proposed_signatures": json.dumps([to_jsonable_python(info2)])},
]
)
dspy.settings.configure(lm=qa_model)
diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.py
index bdc9122c7..2451bef66 100644
--- a/tests/predict/test_chain_of_thought.py
+++ b/tests/predict/test_chain_of_thought.py
@@ -6,7 +6,7 @@ from dspy.utils import DummyLM
def test_initialization_with_string_signature():
- lm = DummyLM(["[[ ## reasoning ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])
+ lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])
dspy.settings.configure(lm=lm)
predict = ChainOfThought("question -> answer")
assert list(predict.extended_signature.output_fields.keys()) == [
@@ -14,44 +14,3 @@ def test_initialization_with_string_signature():
"answer",
]
assert predict(question="What is 1+1?").answer == "2"
-
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
-
- Your output fields are:
- 1. `reasoning` (str)
- 2. `answer` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## answer ## ]]
- {answer}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `question`, produce the fields `answer`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
- ),
- },
- ]
diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.py
index 3fff5d2cb..77c4f9ac2 100644
--- a/tests/predict/test_chain_of_thought_with_hint.py
+++ b/tests/predict/test_chain_of_thought_with_hint.py
@@ -6,7 +6,7 @@ from dspy.utils import DummyLM
def test_cot_with_no_hint():
- lm = DummyLM(["[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])
+ lm = DummyLM([{"rationale": "find the number after 1", "answer": "2"}])
dspy.settings.configure(lm=lm)
predict = ChainOfThoughtWithHint("question -> answer")
# Check output fields have the right order
@@ -17,54 +17,9 @@ def test_cot_with_no_hint():
]
assert predict(question="What is 1+1?").answer == "2"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
-
- Your output fields are:
- 1. `rationale` (str): ${produce the answer}. We ...
- 2. `answer` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## rationale ## ]]
- {rationale}
-
- [[ ## answer ## ]]
- {answer}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `question`, produce the fields `answer`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- Respond with the corresponding output fields, starting with the field `rationale`, then `answer`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_cot_with_hint():
- lm = DummyLM(
- [
- "[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## hint ## ]]\nIs it helicopter?\n\n[[ ## answer ## ]]\n2"
- ]
- )
+ lm = DummyLM([{"rationale": "find the number after 1", "hint": "Is it helicopter?", "answer": "2"}])
dspy.settings.configure(lm=lm)
predict = ChainOfThoughtWithHint("question -> answer")
assert list(predict.extended_signature2.output_fields.keys()) == [
@@ -73,48 +28,3 @@ def test_cot_with_hint():
"answer",
]
assert predict(question="What is 1+1?", hint="think small").answer == "2"
-
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
-
- Your output fields are:
- 1. `rationale` (str): ${produce the answer}. We ...
- 2. `hint` (str)
- 3. `answer` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## rationale ## ]]
- {rationale}
-
- [[ ## hint ## ]]
- {hint}
-
- [[ ## answer ## ]]
- {answer}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `question`, produce the fields `answer`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- Respond with the corresponding output fields, starting with the field `rationale`, then `hint`, then `answer`, and then ending with the marker for `completed`."""
- ),
- },
- ]
diff --git a/tests/predict/test_multi_chain_comparison.py b/tests/predict/test_multi_chain_comparison.py
index 87e2af1a7..8354f64d0 100644
--- a/tests/predict/test_multi_chain_comparison.py
+++ b/tests/predict/test_multi_chain_comparison.py
@@ -30,7 +30,7 @@ def test_basic_example():
# Call the MultiChainComparison on the completions
question = "What is the color of the sky?"
- lm = DummyLM(["[[ ## rationale ## ]]\nmy rationale\n\n[[ ## answer ## ]]\nblue"])
+ lm = DummyLM([{"rationale": "my rationale", "answer": "blue"}])
dspy.settings.configure(lm=lm)
final_pred = compare_answers(completions, question=question)
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index cf499474e..b04e087e8 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -41,48 +41,11 @@ def test_lm_after_dump_and_load_state():
def test_call_method():
predict_instance = Predict("input -> output")
- lm = DummyLM(["[[ ## output ## ]]\ntest output"])
+ lm = DummyLM([{"output": "test output"}])
dspy.settings.configure(lm=lm)
result = predict_instance(input="test input")
assert result.output == "test output"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `input` (str)
-
- Your output fields are:
- 1. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## input ## ]]
- {input}
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `input`, produce the fields `output`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- test input
-
- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_instructions_after_dump_and_load_state():
predict_instance = Predict(Signature("input -> output", "original instructions"))
@@ -166,16 +129,14 @@ def test_typed_demos_after_dump_and_load_state():
def test_forward_method():
program = Predict("question -> answer")
- dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nNo more responses"]))
+ dspy.settings.configure(lm=DummyLM([{"answer": "No more responses"}]))
result = program(question="What is 1+1?").answer
assert result == "No more responses"
def test_forward_method2():
program = Predict("question -> answer1, answer2")
- dspy.settings.configure(
- lm=DummyLM(["[[ ## answer1 ## ]]\nmy first answer\n\n[[ ## answer2 ## ]]\nmy second answer"])
- )
+ dspy.settings.configure(lm=DummyLM([{"answer1": "my first answer", "answer2": "my second answer"}]))
result = program(question="What is 1+1?")
assert result.answer1 == "my first answer"
assert result.answer2 == "my second answer"
@@ -190,7 +151,7 @@ def test_config_management():
def test_multi_output():
program = Predict("question -> answer", n=2)
- dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nmy first answer", "[[ ## answer ## ]]\nmy second answer"]))
+ dspy.settings.configure(lm=DummyLM([{"answer": "my first answer"}, {"answer": "my second answer"}]))
results = program(question="What is 1+1?")
assert results.completions.answer[0] == "my first answer"
assert results.completions.answer[1] == "my second answer"
@@ -201,8 +162,8 @@ def test_multi_output2():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## answer1 ## ]]\nmy 0 answer\n\n[[ ## answer2 ## ]]\nmy 2 answer",
- "[[ ## answer1 ## ]]\nmy 1 answer\n\n[[ ## answer2 ## ]]\nmy 3 answer",
+ {"answer1": "my 0 answer", "answer2": "my 2 answer"},
+ {"answer1": "my 1 answer", "answer2": "my 3 answer"},
],
)
)
@@ -233,39 +194,6 @@ def test_output_only():
predictor = Predict(OutputOnlySignature)
- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])
+ lm = DummyLM([{"output": "short answer"}])
dspy.settings.configure(lm=lm)
assert predictor().output == "short answer"
-
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
-
-
- Your output fields are:
- 1. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
-
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields , produce the fields `output`."""
- ),
- },
- {
- "role": "user",
- "content": (
- "Respond with the corresponding output fields, "
- "starting with the field `output`, and then ending with the marker for `completed`."
- ),
- },
- ]
diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.py
index 5bb654753..007c4e664 100644
--- a/tests/predict/test_program_of_thought.py
+++ b/tests/predict/test_program_of_thought.py
@@ -13,8 +13,8 @@ class BasicQA(Signature):
def test_pot_code_generation():
lm = DummyLM(
[
- "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",
- "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## answer ## ]]\n2",
+ {"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+1\n```"},
+ {"reasoning": "Reason_B", "answer": "2"},
]
)
dspy.settings.configure(lm=lm)
@@ -22,68 +22,13 @@ def test_pot_code_generation():
res = pot(question="What is 1+1?")
assert res.answer == "2"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
- 2. `final_generated_code` (str): python code that answers the question
- 3. `code_output` (str): output of previously-generated python code
-
- Your output fields are:
- 1. `reasoning` (str)
- 2. `answer` (str): often between 1 and 5 words
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## final_generated_code ## ]]
- {final_generated_code}
-
- [[ ## code_output ## ]]
- {code_output}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## answer ## ]]
- {answer}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- [[ ## final_generated_code ## ]]
- result = 1+1
-
- [[ ## code_output ## ]]
- 2
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_pot_code_generation_with_error():
lm = DummyLM(
[
- "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+0/0\n```",
- "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",
- "[[ ## reasoning ## ]]\nReason_C\n\n[[ ## answer ## ]]\n2",
+ {"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+0/0\n```"},
+ {"reasoning": "Reason_B", "generated_code": "```python\nresult = 1+1\n```"},
+ {"reasoning": "Reason_C", "answer": "2"},
]
)
dspy.settings.configure(lm=lm)
@@ -91,114 +36,3 @@ def test_pot_code_generation_with_error():
pot = ProgramOfThought(BasicQA)
res = pot(question="What is 1+1?")
assert res.answer == "2"
-
- # The first code example failed
- assert lm.get_convo(1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
- 2. `previous_code` (str): previously-generated python code that errored
- 3. `error` (str): error message from previously-generated python code
-
- Your output fields are:
- 1. `reasoning` (str)
- 2. `generated_code` (str): python code that answers the question
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## previous_code ## ]]
- {previous_code}
-
- [[ ## error ## ]]
- {error}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## generated_code ## ]]
- {generated_code}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- You are given `question`, `previous_code`, `error` due to an error in previous code.
- Your task is to correct the error and provide the new `generated_code`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- [[ ## previous_code ## ]]
- result = 1+0/0
-
- [[ ## error ## ]]
- division by zero
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `generated_code`, and then ending with the marker for `completed`."""
- ),
- },
- ]
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `question` (str)
- 2. `final_generated_code` (str): python code that answers the question
- 3. `code_output` (str): output of previously-generated python code
-
- Your output fields are:
- 1. `reasoning` (str)
- 2. `answer` (str): often between 1 and 5 words
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## question ## ]]
- {question}
-
- [[ ## final_generated_code ## ]]
- {final_generated_code}
-
- [[ ## code_output ## ]]
- {code_output}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## answer ## ]]
- {answer}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## question ## ]]
- What is 1+1?
-
- [[ ## final_generated_code ## ]]
- result = 1+1
-
- [[ ## code_output ## ]]
- 2
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""
- ),
- },
- ]
diff --git a/tests/predict/test_react.py b/tests/predict/test_react.py
index 8ee626aeb..2573abcb8 100644
--- a/tests/predict/test_react.py
+++ b/tests/predict/test_react.py
@@ -8,7 +8,7 @@ def test_example_no_tools():
# Createa a simple dataset which the model will use with the Retrieve tool.
lm = dspy.utils.DummyLM(
[
- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nFinish[blue]",
+ {"Thought_1": "Initial thoughts", "Action_1": "Finish[blue]"},
]
)
dspy.settings.configure(lm=lm, rm=dummy_rm())
@@ -28,8 +28,8 @@ def test_example_search():
# Createa a simple dataset which the model will use with the Retrieve tool.
lm = dspy.utils.DummyLM(
[
- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nSearch[the color of the sky]",
- "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nFinish[blue]\n\n",
+ {"Thought_1": "Initial thoughts", "Action_1": "Search[the color of the sky]"},
+ {"Thought_2": "More thoughts", "Action_2": "Finish[blue]\n\n"},
]
)
rm = dummy_rm(
@@ -91,9 +91,9 @@ class DummyTool2:
def test_custom_tools():
lm = dspy.utils.DummyLM(
[
- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nTool1[foo]",
- "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nTool2[bar]",
- "[[ ## Thought_3 ## ]]\nEven more thoughts\n\n[[ ## Action_3 ## ]]\nFinish[baz]",
+ {"Thought_1": "Initial thoughts", "Action_1": "Tool1[foo]"},
+ {"Thought_2": "More thoughts", "Action_2": "Tool2[bar]"},
+ {"Thought_3": "Even more thoughts", "Action_3": "Finish[baz]"},
]
)
dspy.settings.configure(lm=lm)
diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.py
index 16050e355..687a18dbf 100644
--- a/tests/predict/test_retry.py
+++ b/tests/predict/test_retry.py
@@ -1,8 +1,10 @@
import functools
+
+import pydantic
+
import dspy
-from dspy.utils import DummyLM
from dspy.primitives.assertions import assert_transform_module, backtrack_handler
-import pydantic
+from dspy.utils import DummyLM
def test_retry_simple():
@@ -14,7 +16,7 @@ def test_retry_simple():
assert f"past_{field}" in retry_module.new_signature.input_fields
assert "feedback" in retry_module.new_signature.input_fields
- lm = DummyLM(["[[ ## answer ## ]]\nblue"])
+ lm = DummyLM([{"answer": "blue"}])
dspy.settings.configure(lm=lm)
result = retry_module.forward(
question="What color is the sky?",
@@ -26,7 +28,7 @@ def test_retry_simple():
def test_retry_forward_with_feedback():
# First we make a mistake, then we fix it
- lm = DummyLM(["[[ ## answer ## ]]\nred", "[[ ## answer ## ]]\nblue"])
+ lm = DummyLM([{"answer": "red"}, {"answer": "blue"}])
dspy.settings.configure(lm=lm, trace=[])
class SimpleModule(dspy.Module):
@@ -53,7 +55,7 @@ def test_retry_forward_with_feedback():
def test_retry_forward_with_typed_predictor():
# First we make a mistake, then we fix it
- lm = DummyLM(['[[ ## output ## ]]\n{"answer":"red"}', '[[ ## output ## ]]\n{"answer":"blue"}'])
+ lm = DummyLM([{"output": '{"answer":"red"}'}, {"output": '{"answer":"blue"}'}])
dspy.settings.configure(lm=lm, trace=[])
class AnswerQuestion(dspy.Signature):
diff --git a/tests/primitives/test_program.py b/tests/primitives/test_program.py
index 9f271e992..ea6633682 100644
--- a/tests/primitives/test_program.py
+++ b/tests/primitives/test_program.py
@@ -39,8 +39,8 @@ def test_forward():
dspy.settings.configure(
lm=DummyLM(
{
- "What is 1+1?": "[[ ## query ## ]]\nlet me check",
- "let me check": "[[ ## answer ## ]]\n2",
+ "What is 1+1?": {"query": "let me check"},
+ "let me check": {"answer": "2"},
}
)
)
diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.py
index 292bafdf2..8fb9c5ff2 100644
--- a/tests/signatures/test_signature.py
+++ b/tests/signatures/test_signature.py
@@ -182,7 +182,7 @@ def test_insantiating2():
def test_multiline_instructions():
- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])
+ lm = DummyLM([{"output": "short answer"}])
dspy.settings.configure(lm=lm)
class MySignature(Signature):
@@ -195,38 +195,6 @@ def test_multiline_instructions():
predictor = dspy.Predict(MySignature)
assert predictor().output == "short answer"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
-
-
- Your output fields are:
- 1. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
-
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- First line
- Second line
- Third line"""
- ),
- },
- {
- "role": "user",
- "content": "Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",
- },
- ]
-
def test_replaced_by_replace_context_manager():
class SignatureOne(Signature):
diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.py
index ebf23f3b4..a112947a4 100644
--- a/tests/teleprompt/test_bootstrap.py
+++ b/tests/teleprompt/test_bootstrap.py
@@ -59,9 +59,7 @@ def test_bootstrap_effectiveness():
# This test verifies if the bootstrapping process improves the student's predictions
student = SimpleModule("input -> output")
teacher = SimpleModule("input -> output")
- lm = DummyLM(
- ["[[ ## output ## ]]\nblue", "[[ ## output ## ]]\nRing-ding-ding-ding-dingeringeding!"], follow_examples=True
- )
+ lm = DummyLM([{"output": "blue"}, {"output": "Ring-ding-ding-ding-dingeringeding!"}], follow_examples=True)
dspy.settings.configure(lm=lm, trace=[])
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
@@ -80,63 +78,6 @@ def test_bootstrap_effectiveness():
prediction = compiled_student(input=trainset[0].input)
assert prediction.output == trainset[0].output
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `input` (str)
-
- Your output fields are:
- 1. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## input ## ]]
- {input}
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Given the fields `input`, produce the fields `output`."""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- What is the color of the sky?
-
- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
- ),
- },
- {
- "role": "assistant",
- "content": textwrap.dedent(
- """\
- [[ ## output ## ]]
- blue
-
- [[ ## completed ## ]]"""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- What is the color of the sky?
-
- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_error_handling_during_bootstrap():
"""
@@ -157,7 +98,7 @@ def test_error_handling_during_bootstrap():
# Setup DummyLM to simulate an error scenario
lm = DummyLM(
[
- "[[ ## output ## ]]\nInitial thoughts", # Simulate initial teacher's prediction
+ {"output": "Initial thoughts"}, # Simulate initial teacher's prediction
]
)
dspy.settings.configure(lm=lm)
@@ -182,8 +123,8 @@ def test_validation_set_usage():
lm = DummyLM(
[
- "[[ ## output ## ]]\nInitial thoughts",
- "[[ ## output ## ]]\nFinish[blue]", # Expected output for both training and validation
+ {"output": "Initial thoughts"},
+ {"output": "Finish[blue]"}, # Expected output for both training and validation
]
)
dspy.settings.configure(lm=lm)
diff --git a/tests/teleprompt/test_copro_optimizer.py b/tests/teleprompt/test_copro_optimizer.py
index 301106dae..c031d3c7a 100644
--- a/tests/teleprompt/test_copro_optimizer.py
+++ b/tests/teleprompt/test_copro_optimizer.py
@@ -1,5 +1,3 @@
-import textwrap
-
import dspy
from dspy import Example
from dspy.teleprompt.signature_opt import COPRO
@@ -44,8 +42,10 @@ def test_signature_optimizer_optimization_process():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## proposed_instruction ## ]]\nOptimized instruction 1\n\n"
- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized instruction 2",
+ {
+ "proposed_instruction": "Optimized instruction 1",
+ "proposed_prefix_for_output_field": "Optimized instruction 2",
+ },
]
)
)
@@ -72,8 +72,10 @@ def test_signature_optimizer_statistics_tracking():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## proposed_instruction ## ]]\nOptimized instruction 1\n\n"
- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized instruction 2",
+ {
+ "proposed_instruction": "Optimized instruction 1",
+ "proposed_prefix_for_output_field": "Optimized instruction 2",
+ },
]
)
)
@@ -93,14 +95,14 @@ def test_signature_optimizer_statistics_tracking():
def test_optimization_and_output_verification():
lm = DummyLM(
[
- "[[ ## proposed_instruction ## ]]\nOptimized Prompt\n\n[[ ## proposed_prefix_for_output_field ## ]]\nOptimized Prefix",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",
+ {"proposed_instruction": "Optimized Prompt", "proposed_prefix_for_output_field": "Optimized Prefix"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
+ {"reasoning": "france", "output": "Paris"},
]
)
dspy.settings.configure(lm=lm)
@@ -121,54 +123,12 @@ def test_optimization_and_output_verification():
assert prediction.output == "Paris"
- assert lm.get_convo(-1)[0] == [
- {
- "role": "system",
- "content": textwrap.dedent(
- """\
- Your input fields are:
- 1. `input` (str)
-
- Your output fields are:
- 1. `reasoning` (str)
- 2. `output` (str)
-
- All interactions will be structured in the following way, with the appropriate values filled in.
-
- [[ ## input ## ]]
- {input}
-
- [[ ## reasoning ## ]]
- {reasoning}
-
- [[ ## output ## ]]
- {output}
-
- [[ ## completed ## ]]
-
- In adhering to this structure, your objective is:
- Optimized Prompt"""
- ),
- },
- {
- "role": "user",
- "content": textwrap.dedent(
- """\
- [[ ## input ## ]]
- What is the capital of France?
-
- Respond with the corresponding output fields, starting with the field `reasoning`, then `output`, and then ending with the marker for `completed`."""
- ),
- },
- ]
-
def test_statistics_tracking_during_optimization():
dspy.settings.configure(
lm=DummyLM(
[
- "[[ ## proposed_instruction ## ]]\nOptimized Prompt\n\n"
- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized Prefix",
+ {"proposed_instruction": "Optimized Prompt", "proposed_prefix_for_output_field": "Optimized Prefix"},
]
)
)