DSPy: The framework for programming—not prompting—language models
Python
feat(dspy): update DummyLM to use ChatAdapter's format_field to format outputs
Michael Jones738ea87a348a
diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.pyindex 3288facf5..bdc51f0aa 100644--- a/dspy/utils/dummies.py+++ b/dspy/utils/dummies.py@@ -7,7 +7,7 @@ import numpy as npfrom dsp.modules import LM as DSPLMfrom dsp.utils.utils import dotdict-from dspy.adapters.chat_adapter import field_header_pattern+from dspy.adapters.chat_adapter import field_header_pattern, format_fieldsfrom dspy.clients.lm import LM@@ -98,7 +98,7 @@ class DSPDummyLM(DSPLM):class DummyLM(LM):- def __init__(self, answers: Union[list[str], dict[str, str]], follow_examples: bool = False):+ def __init__(self, answers: Union[list[dict[str, str]], dict[str, dict[str, str]]], follow_examples: bool = False):super().__init__("dummy", "chat", 0.0, 1000, True)self.answers = answersif isinstance(answers, list):@@ -133,10 +133,13 @@ class DummyLM(LM):outputs.append(self._use_example(messages))elif isinstance(self.answers, dict):outputs.append(- next((v for k, v in self.answers.items() if k in messages[-1]["content"]), "No more responses")+ next(+ (format_fields(v) for k, v in self.answers.items() if k in messages[-1]["content"]),+ "No more responses",+ ))else:- outputs.append(next(self.answers, "No more responses"))+ outputs.append(format_fields(next(self.answers, {"answer": "No more responses"})))# Logging, with removed api key & where `cost` is None on cache hit.kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.pyindex 01c3e5739..a8552c7aa 100644--- a/tests/evaluate/test_evaluate.py+++ b/tests/evaluate/test_evaluate.py@@ -37,8 +37,8 @@ def test_evaluate_call():dspy.settings.configure(lm=DummyLM({- "What is 1+1?": "[[ ## answer ## ]]\n2",- "What is 2+2?": "[[ ## answer ## ]]\n4",+ "What is 1+1?": {"answer": "2"},+ "What is 2+2?": {"answer": "4"},}))@@ -55,9 +55,7 @@ def test_evaluate_call():def test_multithread_evaluate_call():- dspy.settings.configure(- lm=DummyLM({"What is 1+1?": "[[ ## answer ## ]]\n2", "What is 2+2?": "[[ ## answer ## ]]\n4"})- )+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]program = Predict("question -> answer")assert program(question="What is 1+1?").answer == "2"@@ -80,9 +78,7 @@ def test_multi_thread_evaluate_call_cancelled(monkeypatch):time.sleep(1)return super().__call__(*args, **kwargs)- dspy.settings.configure(- lm=SlowLM({"What is 1+1?": "[[ ## answer ## ]]\n2", "What is 2+2?": "[[ ## answer ## ]]\n4"})- )+ dspy.settings.configure(lm=SlowLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]program = Predict("question -> answer")@@ -112,9 +108,7 @@ def test_multi_thread_evaluate_call_cancelled(monkeypatch):def test_evaluate_call_bad():- dspy.settings.configure(- lm=DummyLM({"What is 1+1?": "[[ ## answer ## ]]\n0", "What is 2+2?": "[[ ## answer ## ]]\n0"})- )+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": {"answer": "0"}, "What is 2+2?": {"answer": "0"}}))devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]program = Predict("question -> answer")ev = Evaluate(diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.pyindex 1f0242763..61ac8c15a 100644--- a/tests/functional/test_functional.py+++ b/tests/functional/test_functional.py@@ -21,7 +21,7 @@ def test_simple():"""Think of a hard factual question about a topic."""expected = "What is the speed of light?"- lm = DummyLM([f"[[ ## hard_question ## ]]\n{expected}"])+ lm = DummyLM([{"hard_question": expected}])dspy.settings.configure(lm=lm)question = hard_question(topic="Physics")@@ -36,7 +36,7 @@ def test_list_output():passexpected = ["What is the speed of light?", "What is the speed of sound?"]- lm = DummyLM(['[[ ## hard_questions ## ]]\n["What is the speed of light?", "What is the speed of sound?"]'])+ lm = DummyLM([{"hard_questions": '["What is the speed of light?", "What is the speed of sound?"]'}])dspy.settings.configure(lm=lm)question = hard_questions(topics=["Physics", "Music"])@@ -54,7 +54,7 @@ def test_simple_type():"""Think of a hard factual question about a topic."""expected = "What is the speed of light?"- lm = DummyLM([f'[[ ## hard_question ## ]]\n{{"value": "{expected}"}}'])+ lm = DummyLM([{"hard_question": f'{{"value": "{expected}"}}'}])dspy.settings.configure(lm=lm)question = hard_question(topic="Physics")@@ -75,7 +75,7 @@ def test_simple_type_input():passquestion = Question(value="What is the speed of light?")- lm = DummyLM([f'[[ ## answer ## ]]\n{{"value": "3e8"}}'])+ lm = DummyLM([{"answer": '{"value": "3e8"}'}])dspy.settings.configure(lm=lm)result = answer(question=question)@@ -110,10 +110,10 @@ def test_simple_class():lm = DummyLM([- "[[ ## hard_question ## ]]\nWhat is the speed of light?",- "[[ ## reasoning ## ]]\nSome bad reasoning, 3e8 m/s.\n\n[[ ## answer ## ]]\n3e8", # Bad answer 1- "[[ ## json_object ## ]]\n{...}", # Model is asked to create an example- f"[[ ## reasoning ## ]]\nSome good reasoning, 3e8 m/s.\n\n[[ ## answer ## ]]\n{expected.model_dump_json()}", # Good answer+ {"hard_question": "What is the speed of light?"},+ {"reasoning": "Some bad reasoning, 3e8 m/s.", "answer": "3e8"}, # Bad answer 1+ {"json_object": "{...}"}, # Model is asked to create an example+ {"reasoning": "Some good reasoning, 3e8 m/s.", "answer": f"{expected.model_dump_json()}"}, # Good answer])dspy.settings.configure(lm=lm)@@ -143,7 +143,7 @@ def test_simple_oop():expected = "What is the speed of light?"lm = DummyLM([- f"[[ ## output ## ]]\n{Question(value=expected).model_dump_json()}",+ {"output": f"{Question(value=expected).model_dump_json()}"},])dspy.settings.configure(lm=lm)@@ -242,24 +242,6 @@ def test_bootstrap_effectiveness():prediction = compiled_student(input=trainset[0].input)assert prediction == trainset[0].output- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": "Your input fields are:\n1. `input` (str)\n\nYour output fields are:\n1. `output` (str)\n\nAll interactions will be structured in the following way, with the appropriate values filled in.\n\n[[ ## input ## ]]\n{input}\n\n[[ ## output ## ]]\n{output}\n\n[[ ## completed ## ]]\n\nIn adhering to this structure, your objective is: \n Given the fields `input`, produce the fields `output`.",- },- {- "role": "user",- "content": "[[ ## input ## ]]\nWhat is the color of the sky?\n\nRespond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",- },- {"role": "assistant", "content": "[[ ## output ## ]]\nblue\n\n[[ ## completed ## ]]"},- {- "role": "user",- "content": "[[ ## input ## ]]\nWhat is the color of the sky?\n\nRespond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",- },- ]-- assert lm.get_convo(-1)[1] == ["[[ ## output ## ]]\nblue\n\n[[ ## completed ## ]]"]-def test_regex():class TravelInformation(BaseModel):@@ -281,11 +263,11 @@ def test_regex():lm = DummyLM([# Example with a bad origin code.- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},# Example to help the model understand- "[[ ## json_object ## ]]\n{...}",+ {"json_object": "{...}"},# Fixed- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}'},])dspy.settings.configure(lm=lm)@@ -341,11 +323,12 @@ def test_custom_model_validate_json():[# Example with a bad origin code.(- "[[ ## flight_information ## ]]\nHere is your json: "- "{"- '"origin": {"code":"JFK", "lat":40.6446, "lon":-73.7797}, '- '"destination": {"code":"LAX", "lat":33.942791, "lon":-118.410042}, '- '"date": "2022-12-25"}'+ {+ "flight_information": "Here is your json: {"+ '"origin": {"code":"JFK", "lat":40.6446, "lon":-73.7797}, '+ '"destination": {"code":"LAX", "lat":33.942791, "lon":-118.410042}, '+ '"date": "2022-12-25"}'+ }),])@@ -370,8 +353,8 @@ def test_raises():lm = DummyLM([- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "bad date"}',+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "bad date"}'},])dspy.settings.configure(lm=lm)@@ -393,11 +376,11 @@ def test_multi_errors():lm = DummyLM([# First origin is wrong, then destination, then all is good- '[[ ## flight_information ## ]]\n{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',- "[[ ## json_object ## ]]\n{...}", # Example to help the model understand- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}',- "[[ ## json_object ## ]]\n{...}", # Example to help the model understand- '[[ ## flight_information ## ]]\n{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',+ {"flight_information": '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}'},+ {"json_object": "{...}"}, # Example to help the model understand+ {"flight_information": '{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}'},+ {"json_object": "{...}"}, # Example to help the model understand+ {"flight_information": '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}'},])dspy.settings.configure(lm=lm)@@ -406,57 +389,6 @@ def test_multi_errors():origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25))- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `email` (str)- 2. `error_flight_information_0` (str): An error to avoid in the future- 3. `error_flight_information_1` (str): An error to avoid in the future-- Your output fields are:- 1. `flight_information` (TravelInformation): ${flight_information}. Respond with a single JSON object. JSON Schema: {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## email ## ]]- {email}-- [[ ## error_flight_information_0 ## ]]- {error_flight_information_0}-- [[ ## error_flight_information_1 ## ]]- {error_flight_information_1}-- [[ ## flight_information ## ]]- {flight_information}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `email`, produce the fields `flight_information`."""- ), # noqa- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## email ## ]]- Some email-- [[ ## error_flight_information_0 ## ]]- String should match pattern '^[A-Z]{3}$': origin (error type: string_pattern_mismatch)-- [[ ## error_flight_information_1 ## ]]- String should match pattern '^[A-Z]{3}$': destination (error type: string_pattern_mismatch)-- Respond with the corresponding output fields, starting with the field `flight_information`, and then ending with the marker for `completed`."""- ),- },- ]-def test_field_validator():class UserDetails(BaseModel):@@ -476,54 +408,12 @@ def test_field_validator():# Keep making the mistake (lower case name) until we run# out of retries.- lm = DummyLM(- [- '[[ ## get_user_details ## ]]\n{"name": "lower case name", "age": 25}',- ]- * 10- )+ lm = DummyLM([{"get_user_details": '{"name": "lower case name", "age": 25}'}] * 10)dspy.settings.configure(lm=lm)with pytest.raises(ValueError):get_user_details()- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `json_schema` (str)-- Your output fields are:- 1. `json_object` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## json_schema ## ]]- {json_schema}-- [[ ## json_object ## ]]- {json_object}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Make a very succinct json object that validates with the following schema"""- ), # noqa- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## json_schema ## ]]- {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}-- Respond with the corresponding output fields, starting with the field `json_object`, and then ending with the marker for `completed`."""- ),- },- ]-def test_annotated_field():@predictor@@ -531,7 +421,7 @@ def test_annotated_field():pass# First try 0, which fails, then try 0.5, which passes- lm = DummyLM(["[[ ## test ## ]]\n0", "[[ ## test ## ]]\n0.5"])+ lm = DummyLM([{"test": "0"}, {"test": "0.5"}])dspy.settings.configure(lm=lm)output = test(input="input")@@ -540,7 +430,7 @@ def test_annotated_field():def test_multiple_outputs():- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])dspy.settings.configure(lm=lm)test = TypedPredictor("input -> output")@@ -549,7 +439,7 @@ def test_multiple_outputs():def test_multiple_outputs_int():- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])dspy.settings.configure(lm=lm)class TestSignature(dspy.Signature):@@ -566,9 +456,9 @@ def test_multiple_outputs_int_cot():# Note: Multiple outputs only work when the language model "speculatively" generates all the outputs in one go.lm = DummyLM([- "[[ ## reasoning ## ]]\nthoughts 0\n\n[[ ## output ## ]]\n0\n",- "[[ ## reasoning ## ]]\nthoughts 1\n\n[[ ## output ## ]]\n1\n",- "[[ ## reasoning ## ]]\nthoughts 2\n\n[[ ## output ## ]]\n2\n",+ {"reasoning": "thoughts 0", "output": "0"},+ {"reasoning": "thoughts 1", "output": "1"},+ {"reasoning": "thoughts 2", "output": "2"},])dspy.settings.configure(lm=lm)@@ -580,7 +470,7 @@ def test_multiple_outputs_int_cot():def test_parse_type_string():- lm = DummyLM([f"[[ ## output ## ]]\n{i}" for i in range(100)])+ lm = DummyLM([{"output": f"{i}"} for i in range(100)])dspy.settings.configure(lm=lm)test = TypedPredictor("input:int -> output:int")@@ -590,7 +480,7 @@ def test_parse_type_string():def test_literal():- lm = DummyLM(['[[ ## f ## ]]\n"2"', '[[ ## f ## ]]\n"3"'])+ lm = DummyLM([{"f": '"2"'}, {"f": '"3"'}])dspy.settings.configure(lm=lm)@predictor@@ -601,7 +491,7 @@ def test_literal():def test_literal_missmatch():- lm = DummyLM([f'[[ ## f ## ]]\n"{i}"' for i in range(5, 100)])+ lm = DummyLM([{"f": f"{i}"} for i in range(5, 100)])dspy.settings.configure(lm=lm)@predictor(max_retries=1)@@ -615,7 +505,7 @@ def test_literal_missmatch():def test_literal_int():- lm = DummyLM(["[[ ## f ## ]]\n2", "[[ ## f ## ]]\n3"])+ lm = DummyLM([{"f": "2"}, {"f": "3"}])dspy.settings.configure(lm=lm)@predictor@@ -626,7 +516,7 @@ def test_literal_int():def test_literal_int_missmatch():- lm = DummyLM([f"[[ ## f ## ]]\n{i}" for i in range(5, 100)])+ lm = DummyLM([{"f": f"{i}"} for i in range(5, 100)])dspy.settings.configure(lm=lm)@predictor(max_retries=1)@@ -645,8 +535,8 @@ def test_fields_on_base_signature():lm = DummyLM([- "[[ ## output ## ]]\n2.1", # Bad output- "[[ ## output ## ]]\n0.5", # Good output+ {"output": "2.1"}, # Bad output+ {"output": "0.5"}, # Good output])dspy.settings.configure(lm=lm)@@ -668,12 +558,12 @@ def test_synthetic_data_gen():lm = DummyLM([- '[[ ## fact ## ]]\n{"fact": "The sky is blue", "varacity": true}',- '[[ ## fact ## ]]\n{"fact": "The sky is green", "varacity": false}',- '[[ ## fact ## ]]\n{"fact": "The sky is red", "varacity": true}',- '[[ ## fact ## ]]\n{"fact": "The earth is flat", "varacity": false}',- '[[ ## fact ## ]]\n{"fact": "The earth is round", "varacity": true}',- '[[ ## fact ## ]]\n{"fact": "The earth is a cube", "varacity": false}',+ {"fact": '{"fact": "The sky is blue", "varacity": true}'},+ {"fact": '{"fact": "The sky is green", "varacity": false}'},+ {"fact": '{"fact": "The sky is red", "varacity": true}'},+ {"fact": '{"fact": "The earth is flat", "varacity": false}'},+ {"fact": '{"fact": "The earth is round", "varacity": true}'},+ {"fact": '{"fact": "The earth is a cube", "varacity": false}'},])dspy.settings.configure(lm=lm)@@ -709,7 +599,7 @@ def test_list_input2():program = TypedChainOfThought(ScoredSignature)- lm = DummyLM(["[[ ## reasoning ## ]]\nThoughts\n\n[[ ## proposed_signature ## ]]\nOutput"])+ lm = DummyLM([{"reasoning": "Thoughts", "proposed_signature": "Output"}])dspy.settings.configure(lm=lm)output = program(@@ -724,49 +614,6 @@ def test_list_input2():assert output == "Output"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `attempted_signatures` (list[ScoredString])-- Your output fields are:- 1. `reasoning` (str): ${produce the proposed_signature}. We ...- 2. `proposed_signature` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## attempted_signatures ## ]]- {attempted_signatures}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## proposed_signature ## ]]- {proposed_signature}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `attempted_signatures`, produce the fields `proposed_signature`."""- ), # noqa- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## attempted_signatures ## ]]- [1] «string='string 1' score=0.5»- [2] «string='string 2' score=0.4»- [3] «string='string 3' score=0.3»-- Respond with the corresponding output fields, starting with the field `reasoning`, then `proposed_signature`, and then ending with the marker for `completed`."""- ),- },- ]-def test_custom_reasoning_field():class Question(pydantic.BaseModel):@@ -784,7 +631,7 @@ def test_custom_reasoning_field():program = TypedChainOfThought(QuestionSignature, reasoning=reasoning)expected = "What is the speed of light?"- lm = DummyLM([f'[[ ## reasoning ## ]]\nThoughts\n\n[[ ## question ## ]]\n{{"value": "{expected}"}}'])+ lm = DummyLM([{"reasoning": "Thoughts", "question": f'{{"value": "{expected}"}}'}])dspy.settings.configure(lm=lm)output = program(topic="Physics")@@ -792,47 +639,6 @@ def test_custom_reasoning_field():assert isinstance(output.question, Question)assert output.question.value == expected- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `topic` (str)-- Your output fields are:- 1. `reasoning` (str): ${topic}, we should ...- 2. `question` (Question): ${question}. Respond with a single JSON object. JSON Schema: {"properties": {"value": {"title": "Value", "type": "string"}}, "required": ["value"], "title": "Question", "type": "object"}-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## topic ## ]]- {topic}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## question ## ]]- {question}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `topic`, produce the fields `question`."""- ), # noqa- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## topic ## ]]- Physics-- Respond with the corresponding output fields, starting with the field `reasoning`, then `question`, and then ending with the marker for `completed`."""- ),- },- ]-def test_generic_signature():T = TypeVar("T")@@ -845,7 +651,7 @@ def test_generic_signature():predictor = TypedPredictor(GenericSignature[int])assert predictor.signature.instructions == "My signature"- lm = DummyLM(["[[ ## output ## ]]\n23"])+ lm = DummyLM([{"output": "23"}])dspy.settings.configure(lm=lm)assert predictor().output == 23@@ -858,7 +664,7 @@ def test_field_validator_in_signature():@pydantic.field_validator("a")@classmethoddef space_in_a(cls, a: str) -> str:- if not " " in a:+ if " " not in a:raise ValueError("a must contain a space")return a@@ -883,10 +689,10 @@ def test_lm_as_validator():lm = DummyLM([- "[[ ## next_square ## ]]\n3",- "[[ ## is_square ## ]]\nFalse",- "[[ ## next_square ## ]]\n4",- "[[ ## is_square ## ]]\nTrue",+ {"next_square": "3"},+ {"is_square": "False"},+ {"next_square": "4"},+ {"is_square": "True"},])dspy.settings.configure(lm=lm)@@ -909,7 +715,7 @@ def test_annotated_validator():n: int = dspy.InputField()next_square: Annotated[int, AfterValidator(is_square)] = dspy.OutputField()- lm = DummyLM(["[[ ## next_square ## ]]\n3", "[[ ## next_square ## ]]\n4"])+ lm = DummyLM([{"next_square": "3"}, {"next_square": "4"}])dspy.settings.configure(lm=lm)m = TypedPredictor(MySignature)(n=2).next_square@@ -928,7 +734,7 @@ def test_annotated_validator_functional():def next_square(n: int) -> Annotated[int, AfterValidator(is_square)]:"""What is the next square number after n?"""- lm = DummyLM(["[[ ## next_square ## ]]\n3", "[[ ## next_square ## ]]\n4"])+ lm = DummyLM([{"next_square": "3"}, {"next_square": "4"}])dspy.settings.configure(lm=lm)m = next_square(n=2)@@ -946,68 +752,11 @@ def test_demos():trainset=[ex.with_inputs("input") for ex in demos],)- lm = DummyLM(["[[ ## output ## ]]\nParis"])+ lm = DummyLM([{"output": "Paris"}])dspy.settings.configure(lm=lm)assert program(input="What is the capital of France?").output == "Paris"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `input` (str)-- Your output fields are:- 1. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## input ## ]]- {input}-- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `input`, produce the fields `output`."""- ), # noqa- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- What is the speed of light?-- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""- ),- },- {- "role": "assistant",- "content": textwrap.dedent(- """\- [[ ## output ## ]]- 3e8-- [[ ## completed ## ]]"""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- What is the capital of France?-- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""- ),- },- ]-def test_demos_missing_input_in_demo():demos = [dspy.Example(input="What is the speed of light?", output="3e8")]@@ -1015,7 +764,7 @@ def test_demos_missing_input_in_demo():student=dspy.TypedPredictor("input -> output, thoughts"),trainset=[ex.with_inputs("input") for ex in demos],)- lm = DummyLM(["[[ ## thoughts ## ]]\nMy thoughts\n\n[[ ## output ## ]]\nParis"])+ lm = DummyLM([{"thoughts": "My thoughts", "output": "Paris"}])dspy.settings.configure(lm=lm)assert program(input="What is the capital of France?").output == "Paris"@@ -1024,10 +773,10 @@ def test_conlist():dspy.settings.configure(lm=DummyLM([- "[[ ## make_numbers ## ]]\n[]",- "[[ ## make_numbers ## ]]\n[1]",- "[[ ## make_numbers ## ]]\n[1, 2]",- "[[ ## make_numbers ## ]]\n[1, 2, 3]",+ {"make_numbers": "[]"},+ {"make_numbers": "[1]"},+ {"make_numbers": "[1, 2]"},+ {"make_numbers": "[1, 2, 3]"},]))@@ -1043,10 +792,10 @@ def test_conlist2():dspy.settings.configure(lm=DummyLM([- "[[ ## output ## ]]\n[]",- "[[ ## output ## ]]\n[1]",- "[[ ## output ## ]]\n[1, 2]",- "[[ ## output ## ]]\n[1, 2, 3]",+ {"output": "[]"},+ {"output": "[1]"},+ {"output": "[1, 2]"},+ {"output": "[1, 2, 3]"},]))@@ -1067,7 +816,7 @@ def test_model_validator():raise ValueError(f"category not in {self.allowed_categories}")return self- lm = DummyLM(["[[ ## category ## ]]\nhorse", "[[ ## category ## ]]\ndog"])+ lm = DummyLM([{"category": "horse"}, {"category": "dog"}])dspy.settings.configure(lm=lm)predictor = TypedPredictor(MySignature)diff --git a/tests/functional/test_signature_opt_typed.py b/tests/functional/test_signature_opt_typed.pyindex 125cec30b..6778f6b69 100644--- a/tests/functional/test_signature_opt_typed.py+++ b/tests/functional/test_signature_opt_typed.py@@ -1,7 +1,5 @@import json-from typing import Generic, TypeVar-import pydanticfrom pydantic_core import to_jsonable_pythonimport dspy@@ -103,12 +101,14 @@ def test_opt():question: str = dspy.InputField()answer: str = dspy.OutputField()- qa_model = DummyLM(["[[ ## answer ## ]]\nfoo"] * 100)+ qa_model = DummyLM([{"answer": "foo"}] * 100)prompt_model = DummyLM([+ {+ "reasoning": "some thoughts",+ "proposed_signatures": '[{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]',+ }# Seed prompts- "[[ ## reasoning ## ]]\nsome thoughts\n\n"- '[[ ## proposed_signatures ## ]]\n[{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]',])dspy.settings.configure(lm=qa_model)@@ -165,8 +165,8 @@ def test_opt_composed():qa_model = DummyLM([])prompt_model = DummyLM([- f"[[ ## reasoning ## ]]\nsome thoughts\n\n[[ ## proposed_signatures ## ]]\n{json.dumps([to_jsonable_python(info1)])}",- f"[[ ## reasoning ## ]]\nsome thoughts\n\n[[ ## proposed_signatures ## ]]\n{json.dumps([to_jsonable_python(info2)])}",+ {"reasoning": "some thoughts", "proposed_signatures": json.dumps([to_jsonable_python(info1)])},+ {"reasoning": "some thoughts", "proposed_signatures": json.dumps([to_jsonable_python(info2)])},])dspy.settings.configure(lm=qa_model)diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pyindex bdc9122c7..2451bef66 100644--- a/tests/predict/test_chain_of_thought.py+++ b/tests/predict/test_chain_of_thought.py@@ -6,7 +6,7 @@ from dspy.utils import DummyLMdef test_initialization_with_string_signature():- lm = DummyLM(["[[ ## reasoning ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])+ lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])dspy.settings.configure(lm=lm)predict = ChainOfThought("question -> answer")assert list(predict.extended_signature.output_fields.keys()) == [@@ -14,44 +14,3 @@ def test_initialization_with_string_signature():"answer",]assert predict(question="What is 1+1?").answer == "2"-- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)-- Your output fields are:- 1. `reasoning` (str)- 2. `answer` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## answer ## ]]- {answer}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `question`, produce the fields `answer`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""- ),- },- ]diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.pyindex 3fff5d2cb..77c4f9ac2 100644--- a/tests/predict/test_chain_of_thought_with_hint.py+++ b/tests/predict/test_chain_of_thought_with_hint.py@@ -6,7 +6,7 @@ from dspy.utils import DummyLMdef test_cot_with_no_hint():- lm = DummyLM(["[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## answer ## ]]\n2"])+ lm = DummyLM([{"rationale": "find the number after 1", "answer": "2"}])dspy.settings.configure(lm=lm)predict = ChainOfThoughtWithHint("question -> answer")# Check output fields have the right order@@ -17,54 +17,9 @@ def test_cot_with_no_hint():]assert predict(question="What is 1+1?").answer == "2"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)-- Your output fields are:- 1. `rationale` (str): ${produce the answer}. We ...- 2. `answer` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## rationale ## ]]- {rationale}-- [[ ## answer ## ]]- {answer}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `question`, produce the fields `answer`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- Respond with the corresponding output fields, starting with the field `rationale`, then `answer`, and then ending with the marker for `completed`."""- ),- },- ]-def test_cot_with_hint():- lm = DummyLM(- [- "[[ ## rationale ## ]]\nfind the number after 1\n\n[[ ## hint ## ]]\nIs it helicopter?\n\n[[ ## answer ## ]]\n2"- ]- )+ lm = DummyLM([{"rationale": "find the number after 1", "hint": "Is it helicopter?", "answer": "2"}])dspy.settings.configure(lm=lm)predict = ChainOfThoughtWithHint("question -> answer")assert list(predict.extended_signature2.output_fields.keys()) == [@@ -73,48 +28,3 @@ def test_cot_with_hint():"answer",]assert predict(question="What is 1+1?", hint="think small").answer == "2"-- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)-- Your output fields are:- 1. `rationale` (str): ${produce the answer}. We ...- 2. `hint` (str)- 3. `answer` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## rationale ## ]]- {rationale}-- [[ ## hint ## ]]- {hint}-- [[ ## answer ## ]]- {answer}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `question`, produce the fields `answer`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- Respond with the corresponding output fields, starting with the field `rationale`, then `hint`, then `answer`, and then ending with the marker for `completed`."""- ),- },- ]diff --git a/tests/predict/test_multi_chain_comparison.py b/tests/predict/test_multi_chain_comparison.pyindex 87e2af1a7..8354f64d0 100644--- a/tests/predict/test_multi_chain_comparison.py+++ b/tests/predict/test_multi_chain_comparison.py@@ -30,7 +30,7 @@ def test_basic_example():# Call the MultiChainComparison on the completionsquestion = "What is the color of the sky?"- lm = DummyLM(["[[ ## rationale ## ]]\nmy rationale\n\n[[ ## answer ## ]]\nblue"])+ lm = DummyLM([{"rationale": "my rationale", "answer": "blue"}])dspy.settings.configure(lm=lm)final_pred = compare_answers(completions, question=question)diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.pyindex cf499474e..b04e087e8 100644--- a/tests/predict/test_predict.py+++ b/tests/predict/test_predict.py@@ -41,48 +41,11 @@ def test_lm_after_dump_and_load_state():def test_call_method():predict_instance = Predict("input -> output")- lm = DummyLM(["[[ ## output ## ]]\ntest output"])+ lm = DummyLM([{"output": "test output"}])dspy.settings.configure(lm=lm)result = predict_instance(input="test input")assert result.output == "test output"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `input` (str)-- Your output fields are:- 1. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## input ## ]]- {input}-- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `input`, produce the fields `output`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- test input-- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""- ),- },- ]-def test_instructions_after_dump_and_load_state():predict_instance = Predict(Signature("input -> output", "original instructions"))@@ -166,16 +129,14 @@ def test_typed_demos_after_dump_and_load_state():def test_forward_method():program = Predict("question -> answer")- dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nNo more responses"]))+ dspy.settings.configure(lm=DummyLM([{"answer": "No more responses"}]))result = program(question="What is 1+1?").answerassert result == "No more responses"def test_forward_method2():program = Predict("question -> answer1, answer2")- dspy.settings.configure(- lm=DummyLM(["[[ ## answer1 ## ]]\nmy first answer\n\n[[ ## answer2 ## ]]\nmy second answer"])- )+ dspy.settings.configure(lm=DummyLM([{"answer1": "my first answer", "answer2": "my second answer"}]))result = program(question="What is 1+1?")assert result.answer1 == "my first answer"assert result.answer2 == "my second answer"@@ -190,7 +151,7 @@ def test_config_management():def test_multi_output():program = Predict("question -> answer", n=2)- dspy.settings.configure(lm=DummyLM(["[[ ## answer ## ]]\nmy first answer", "[[ ## answer ## ]]\nmy second answer"]))+ dspy.settings.configure(lm=DummyLM([{"answer": "my first answer"}, {"answer": "my second answer"}]))results = program(question="What is 1+1?")assert results.completions.answer[0] == "my first answer"assert results.completions.answer[1] == "my second answer"@@ -201,8 +162,8 @@ def test_multi_output2():dspy.settings.configure(lm=DummyLM([- "[[ ## answer1 ## ]]\nmy 0 answer\n\n[[ ## answer2 ## ]]\nmy 2 answer",- "[[ ## answer1 ## ]]\nmy 1 answer\n\n[[ ## answer2 ## ]]\nmy 3 answer",+ {"answer1": "my 0 answer", "answer2": "my 2 answer"},+ {"answer1": "my 1 answer", "answer2": "my 3 answer"},],))@@ -233,39 +194,6 @@ def test_output_only():predictor = Predict(OutputOnlySignature)- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])+ lm = DummyLM([{"output": "short answer"}])dspy.settings.configure(lm=lm)assert predictor().output == "short answer"-- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:--- Your output fields are:- 1. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.---- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields , produce the fields `output`."""- ),- },- {- "role": "user",- "content": (- "Respond with the corresponding output fields, "- "starting with the field `output`, and then ending with the marker for `completed`."- ),- },- ]diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.pyindex 5bb654753..007c4e664 100644--- a/tests/predict/test_program_of_thought.py+++ b/tests/predict/test_program_of_thought.py@@ -13,8 +13,8 @@ class BasicQA(Signature):def test_pot_code_generation():lm = DummyLM([- "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",- "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## answer ## ]]\n2",+ {"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+1\n```"},+ {"reasoning": "Reason_B", "answer": "2"},])dspy.settings.configure(lm=lm)@@ -22,68 +22,13 @@ def test_pot_code_generation():res = pot(question="What is 1+1?")assert res.answer == "2"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)- 2. `final_generated_code` (str): python code that answers the question- 3. `code_output` (str): output of previously-generated python code-- Your output fields are:- 1. `reasoning` (str)- 2. `answer` (str): often between 1 and 5 words-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## final_generated_code ## ]]- {final_generated_code}-- [[ ## code_output ## ]]- {code_output}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## answer ## ]]- {answer}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- [[ ## final_generated_code ## ]]- result = 1+1-- [[ ## code_output ## ]]- 2-- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""- ),- },- ]-def test_pot_code_generation_with_error():lm = DummyLM([- "[[ ## reasoning ## ]]\nReason_A\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+0/0\n```",- "[[ ## reasoning ## ]]\nReason_B\n\n[[ ## generated_code ## ]]\n```python\nresult = 1+1\n```",- "[[ ## reasoning ## ]]\nReason_C\n\n[[ ## answer ## ]]\n2",+ {"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+0/0\n```"},+ {"reasoning": "Reason_B", "generated_code": "```python\nresult = 1+1\n```"},+ {"reasoning": "Reason_C", "answer": "2"},])dspy.settings.configure(lm=lm)@@ -91,114 +36,3 @@ def test_pot_code_generation_with_error():pot = ProgramOfThought(BasicQA)res = pot(question="What is 1+1?")assert res.answer == "2"-- # The first code example failed- assert lm.get_convo(1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)- 2. `previous_code` (str): previously-generated python code that errored- 3. `error` (str): error message from previously-generated python code-- Your output fields are:- 1. `reasoning` (str)- 2. `generated_code` (str): python code that answers the question-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## previous_code ## ]]- {previous_code}-- [[ ## error ## ]]- {error}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## generated_code ## ]]- {generated_code}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- You are given `question`, `previous_code`, `error` due to an error in previous code.- Your task is to correct the error and provide the new `generated_code`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- [[ ## previous_code ## ]]- result = 1+0/0-- [[ ## error ## ]]- division by zero-- Respond with the corresponding output fields, starting with the field `reasoning`, then `generated_code`, and then ending with the marker for `completed`."""- ),- },- ]- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `question` (str)- 2. `final_generated_code` (str): python code that answers the question- 3. `code_output` (str): output of previously-generated python code-- Your output fields are:- 1. `reasoning` (str)- 2. `answer` (str): often between 1 and 5 words-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## question ## ]]- {question}-- [[ ## final_generated_code ## ]]- {final_generated_code}-- [[ ## code_output ## ]]- {code_output}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## answer ## ]]- {answer}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## question ## ]]- What is 1+1?-- [[ ## final_generated_code ## ]]- result = 1+1-- [[ ## code_output ## ]]- 2-- Respond with the corresponding output fields, starting with the field `reasoning`, then `answer`, and then ending with the marker for `completed`."""- ),- },- ]diff --git a/tests/predict/test_react.py b/tests/predict/test_react.pyindex 8ee626aeb..2573abcb8 100644--- a/tests/predict/test_react.py+++ b/tests/predict/test_react.py@@ -8,7 +8,7 @@ def test_example_no_tools():# Createa a simple dataset which the model will use with the Retrieve tool.lm = dspy.utils.DummyLM([- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nFinish[blue]",+ {"Thought_1": "Initial thoughts", "Action_1": "Finish[blue]"},])dspy.settings.configure(lm=lm, rm=dummy_rm())@@ -28,8 +28,8 @@ def test_example_search():# Createa a simple dataset which the model will use with the Retrieve tool.lm = dspy.utils.DummyLM([- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nSearch[the color of the sky]",- "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nFinish[blue]\n\n",+ {"Thought_1": "Initial thoughts", "Action_1": "Search[the color of the sky]"},+ {"Thought_2": "More thoughts", "Action_2": "Finish[blue]\n\n"},])rm = dummy_rm(@@ -91,9 +91,9 @@ class DummyTool2:def test_custom_tools():lm = dspy.utils.DummyLM([- "[[ ## Thought_1 ## ]]\nInitial thoughts\n\n[[ ## Action_1 ## ]]\nTool1[foo]",- "[[ ## Thought_2 ## ]]\nMore thoughts\n\n[[ ## Action_2 ## ]]\nTool2[bar]",- "[[ ## Thought_3 ## ]]\nEven more thoughts\n\n[[ ## Action_3 ## ]]\nFinish[baz]",+ {"Thought_1": "Initial thoughts", "Action_1": "Tool1[foo]"},+ {"Thought_2": "More thoughts", "Action_2": "Tool2[bar]"},+ {"Thought_3": "Even more thoughts", "Action_3": "Finish[baz]"},])dspy.settings.configure(lm=lm)diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.pyindex 16050e355..687a18dbf 100644--- a/tests/predict/test_retry.py+++ b/tests/predict/test_retry.py@@ -1,8 +1,10 @@import functools++import pydantic+import dspy-from dspy.utils import DummyLMfrom dspy.primitives.assertions import assert_transform_module, backtrack_handler-import pydantic+from dspy.utils import DummyLMdef test_retry_simple():@@ -14,7 +16,7 @@ def test_retry_simple():assert f"past_{field}" in retry_module.new_signature.input_fieldsassert "feedback" in retry_module.new_signature.input_fields- lm = DummyLM(["[[ ## answer ## ]]\nblue"])+ lm = DummyLM([{"answer": "blue"}])dspy.settings.configure(lm=lm)result = retry_module.forward(question="What color is the sky?",@@ -26,7 +28,7 @@ def test_retry_simple():def test_retry_forward_with_feedback():# First we make a mistake, then we fix it- lm = DummyLM(["[[ ## answer ## ]]\nred", "[[ ## answer ## ]]\nblue"])+ lm = DummyLM([{"answer": "red"}, {"answer": "blue"}])dspy.settings.configure(lm=lm, trace=[])class SimpleModule(dspy.Module):@@ -53,7 +55,7 @@ def test_retry_forward_with_feedback():def test_retry_forward_with_typed_predictor():# First we make a mistake, then we fix it- lm = DummyLM(['[[ ## output ## ]]\n{"answer":"red"}', '[[ ## output ## ]]\n{"answer":"blue"}'])+ lm = DummyLM([{"output": '{"answer":"red"}'}, {"output": '{"answer":"blue"}'}])dspy.settings.configure(lm=lm, trace=[])class AnswerQuestion(dspy.Signature):diff --git a/tests/primitives/test_program.py b/tests/primitives/test_program.pyindex 9f271e992..ea6633682 100644--- a/tests/primitives/test_program.py+++ b/tests/primitives/test_program.py@@ -39,8 +39,8 @@ def test_forward():dspy.settings.configure(lm=DummyLM({- "What is 1+1?": "[[ ## query ## ]]\nlet me check",- "let me check": "[[ ## answer ## ]]\n2",+ "What is 1+1?": {"query": "let me check"},+ "let me check": {"answer": "2"},}))diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.pyindex 292bafdf2..8fb9c5ff2 100644--- a/tests/signatures/test_signature.py+++ b/tests/signatures/test_signature.py@@ -182,7 +182,7 @@ def test_insantiating2():def test_multiline_instructions():- lm = DummyLM(["[[ ## output ## ]]\nshort answer"])+ lm = DummyLM([{"output": "short answer"}])dspy.settings.configure(lm=lm)class MySignature(Signature):@@ -195,38 +195,6 @@ def test_multiline_instructions():predictor = dspy.Predict(MySignature)assert predictor().output == "short answer"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:--- Your output fields are:- 1. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.---- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- First line- Second line- Third line"""- ),- },- {- "role": "user",- "content": "Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`.",- },- ]-def test_replaced_by_replace_context_manager():class SignatureOne(Signature):diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.pyindex ebf23f3b4..a112947a4 100644--- a/tests/teleprompt/test_bootstrap.py+++ b/tests/teleprompt/test_bootstrap.py@@ -59,9 +59,7 @@ def test_bootstrap_effectiveness():# This test verifies if the bootstrapping process improves the student's predictionsstudent = SimpleModule("input -> output")teacher = SimpleModule("input -> output")- lm = DummyLM(- ["[[ ## output ## ]]\nblue", "[[ ## output ## ]]\nRing-ding-ding-ding-dingeringeding!"], follow_examples=True- )+ lm = DummyLM([{"output": "blue"}, {"output": "Ring-ding-ding-ding-dingeringeding!"}], follow_examples=True)dspy.settings.configure(lm=lm, trace=[])bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)@@ -80,63 +78,6 @@ def test_bootstrap_effectiveness():prediction = compiled_student(input=trainset[0].input)assert prediction.output == trainset[0].output- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `input` (str)-- Your output fields are:- 1. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## input ## ]]- {input}-- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Given the fields `input`, produce the fields `output`."""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- What is the color of the sky?-- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""- ),- },- {- "role": "assistant",- "content": textwrap.dedent(- """\- [[ ## output ## ]]- blue-- [[ ## completed ## ]]"""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- What is the color of the sky?-- Respond with the corresponding output fields, starting with the field `output`, and then ending with the marker for `completed`."""- ),- },- ]-def test_error_handling_during_bootstrap():"""@@ -157,7 +98,7 @@ def test_error_handling_during_bootstrap():# Setup DummyLM to simulate an error scenariolm = DummyLM([- "[[ ## output ## ]]\nInitial thoughts", # Simulate initial teacher's prediction+ {"output": "Initial thoughts"}, # Simulate initial teacher's prediction])dspy.settings.configure(lm=lm)@@ -182,8 +123,8 @@ def test_validation_set_usage():lm = DummyLM([- "[[ ## output ## ]]\nInitial thoughts",- "[[ ## output ## ]]\nFinish[blue]", # Expected output for both training and validation+ {"output": "Initial thoughts"},+ {"output": "Finish[blue]"}, # Expected output for both training and validation])dspy.settings.configure(lm=lm)diff --git a/tests/teleprompt/test_copro_optimizer.py b/tests/teleprompt/test_copro_optimizer.pyindex 301106dae..c031d3c7a 100644--- a/tests/teleprompt/test_copro_optimizer.py+++ b/tests/teleprompt/test_copro_optimizer.py@@ -1,5 +1,3 @@-import textwrap-import dspyfrom dspy import Examplefrom dspy.teleprompt.signature_opt import COPRO@@ -44,8 +42,10 @@ def test_signature_optimizer_optimization_process():dspy.settings.configure(lm=DummyLM([- "[[ ## proposed_instruction ## ]]\nOptimized instruction 1\n\n"- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized instruction 2",+ {+ "proposed_instruction": "Optimized instruction 1",+ "proposed_prefix_for_output_field": "Optimized instruction 2",+ },]))@@ -72,8 +72,10 @@ def test_signature_optimizer_statistics_tracking():dspy.settings.configure(lm=DummyLM([- "[[ ## proposed_instruction ## ]]\nOptimized instruction 1\n\n"- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized instruction 2",+ {+ "proposed_instruction": "Optimized instruction 1",+ "proposed_prefix_for_output_field": "Optimized instruction 2",+ },]))@@ -93,14 +95,14 @@ def test_signature_optimizer_statistics_tracking():def test_optimization_and_output_verification():lm = DummyLM([- "[[ ## proposed_instruction ## ]]\nOptimized Prompt\n\n[[ ## proposed_prefix_for_output_field ## ]]\nOptimized Prefix",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",- "[[ ## reasoning ## ]]\nfrance\n\n[[ ## output ## ]]\nParis",+ {"proposed_instruction": "Optimized Prompt", "proposed_prefix_for_output_field": "Optimized Prefix"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},+ {"reasoning": "france", "output": "Paris"},])dspy.settings.configure(lm=lm)@@ -121,54 +123,12 @@ def test_optimization_and_output_verification():assert prediction.output == "Paris"- assert lm.get_convo(-1)[0] == [- {- "role": "system",- "content": textwrap.dedent(- """\- Your input fields are:- 1. `input` (str)-- Your output fields are:- 1. `reasoning` (str)- 2. `output` (str)-- All interactions will be structured in the following way, with the appropriate values filled in.-- [[ ## input ## ]]- {input}-- [[ ## reasoning ## ]]- {reasoning}-- [[ ## output ## ]]- {output}-- [[ ## completed ## ]]-- In adhering to this structure, your objective is:- Optimized Prompt"""- ),- },- {- "role": "user",- "content": textwrap.dedent(- """\- [[ ## input ## ]]- What is the capital of France?-- Respond with the corresponding output fields, starting with the field `reasoning`, then `output`, and then ending with the marker for `completed`."""- ),- },- ]-def test_statistics_tracking_during_optimization():dspy.settings.configure(lm=DummyLM([- "[[ ## proposed_instruction ## ]]\nOptimized Prompt\n\n"- "[[ ## proposed_prefix_for_output_field ## ]]\nOptimized Prefix",+ {"proposed_instruction": "Optimized Prompt", "proposed_prefix_for_output_field": "Optimized Prefix"},]))