DSPy: The framework for programming—not prompting—language models
Python
Add exact adapter format message tests (#9791)
Maxime Rivest52efe77814e1
diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.pyindex 86c662b3b..cfa546660 100644--- a/tests/adapters/test_chat_adapter.py+++ b/tests/adapters/test_chat_adapter.py@@ -99,6 +99,104 @@ async def test_chat_adapter_async_call():assert result == [{"answer": "Paris"}]+def test_chat_adapter_format_exact_messages_for_simple_signature():+ class QA(dspy.Signature):+ """Answer the question."""++ question: str = dspy.InputField()+ answer: str = dspy.OutputField()++ messages = dspy.ChatAdapter().format(QA, [], {"question": "What is the capital of France?"})++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answer` (str):+All interactions will be structured in the following way, with the appropriate values filled in.++[[ ## question ## ]]+{question}++[[ ## answer ## ]]+{answer}++[[ ## completed ## ]]+In adhering to this structure, your objective is:\x20+ Answer the question.""",+ },+ {+ "role": "user",+ "content": """[[ ## question ## ]]+What is the capital of France?++Respond with the corresponding output fields, starting with the field `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.""",+ },+ ]+++def test_chat_adapter_format_exact_messages_with_demo_and_typed_outputs():+ class MultiAnswer(dspy.Signature):+ """Answer the question with multiple answers and scores"""++ question: str = dspy.InputField()+ answers: list[str] = dspy.OutputField()+ scores: list[float] = dspy.OutputField()++ messages = dspy.ChatAdapter().format(+ MultiAnswer,+ demos=[{"question": "Q1", "answers": ["A1", "A2"], "scores": [0.1, 0.9]}],+ inputs={"question": "Q2"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answers` (list[str]):\x20+2. `scores` (list[float]):+All interactions will be structured in the following way, with the appropriate values filled in.++[[ ## question ## ]]+{question}++[[ ## answers ## ]]+{answers} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "string"}}++[[ ## scores ## ]]+{scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}}++[[ ## completed ## ]]+In adhering to this structure, your objective is:\x20+ Answer the question with multiple answers and scores""",+ },+ {"role": "user", "content": """[[ ## question ## ]]+Q1"""},+ {+ "role": "assistant",+ "content": """[[ ## answers ## ]]+["A1", "A2"]++[[ ## scores ## ]]+[0.1, 0.9]++[[ ## completed ## ]]+""",+ },+ {+ "role": "user",+ "content": """[[ ## question ## ]]+Q2++Respond with the corresponding output fields, starting with the field `[[ ## answers ## ]]` (must be formatted as a valid Python list[str]), then `[[ ## scores ## ]]` (must be formatted as a valid Python list[float]), and then ending with the marker for `[[ ## completed ## ]]`.""",+ },+ ]++def test_chat_adapter_with_pydantic_models():"""This test verifies that ChatAdapter can handle different input and output field types, both basic and nested.diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.pyindex f376831d9..13ca0ea81 100644--- a/tests/adapters/test_json_adapter.py+++ b/tests/adapters/test_json_adapter.py@@ -9,6 +9,146 @@ from openai.types.responses import ResponseOutputMessageimport dspy+def test_json_adapter_format_exact_messages_for_simple_signature():+ class StringSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField()++ messages = dspy.JSONAdapter().format(+ StringSignature,+ demos=[],+ inputs={"question": "What is the capital of France?"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answer` (str):+All interactions will be structured in the following way, with the appropriate values filled in.++Inputs will have the following structure:++[[ ## question ## ]]+{question}++Outputs will be a JSON object with the following fields.++{+ "answer": "{answer}"+}+In adhering to this structure, your objective is:\x20+ Given the fields `question`, produce the fields `answer`.""",+ },+ {+ "role": "user",+ "content": """[[ ## question ## ]]+What is the capital of France?++Respond with a JSON object in the following order of fields: `answer`.""",+ },+ ]+++def test_json_adapter_format_exact_messages_with_demo_and_typed_output():+ class MultiAnswer(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField()+ confidence: float = dspy.OutputField()++ messages = dspy.JSONAdapter().format(+ MultiAnswer,+ demos=[{"question": "Q1", "answer": "A1", "confidence": 0.9}],+ inputs={"question": "Q2"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answer` (str):\x20+2. `confidence` (float):+All interactions will be structured in the following way, with the appropriate values filled in.++Inputs will have the following structure:++[[ ## question ## ]]+{question}++Outputs will be a JSON object with the following fields.++{+ "answer": "{answer}",+ "confidence": "{confidence} # note: the value you produce must be a single float value"+}+In adhering to this structure, your objective is:\x20+ Given the fields `question`, produce the fields `answer`, `confidence`.""",+ },+ {"role": "user", "content": """[[ ## question ## ]]+Q1"""},+ {+ "role": "assistant",+ "content": """{+ "answer": "A1",+ "confidence": 0.9+}""",+ },+ {+ "role": "user",+ "content": """[[ ## question ## ]]+Q2++Respond with a JSON object in the following order of fields: `answer`, then `confidence` (must be formatted as a valid Python float).""",+ },+ ]+++def test_json_adapter_format_exact_messages_with_described_and_bool_outputs():+ class TestSignature(dspy.Signature):+ input1: str = dspy.InputField()+ output1: str = dspy.OutputField(desc="String output field")+ output2: bool = dspy.OutputField()++ messages = dspy.JSONAdapter().format(TestSignature, [], {"input1": "Test input"})++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `input1` (str):+Your output fields are:+1. `output1` (str): String output field+2. `output2` (bool):+All interactions will be structured in the following way, with the appropriate values filled in.++Inputs will have the following structure:++[[ ## input1 ## ]]+{input1}++Outputs will be a JSON object with the following fields.++{+ "output1": "{output1}",+ "output2": "{output2} # note: the value you produce must be True or False"+}+In adhering to this structure, your objective is:\x20+ Given the fields `input1`, produce the fields `output1`, `output2`.""",+ },+ {+ "role": "user",+ "content": """[[ ## input1 ## ]]+Test input++Respond with a JSON object in the following order of fields: `output1`, then `output2` (must be formatted as a valid Python bool).""",+ },+ ]++def test_json_adapter_passes_structured_output_when_supported_by_model():class OutputField3(pydantic.BaseModel):subfield1: int = pydantic.Field(description="Int subfield 1", ge=0, le=10)diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.pyindex ded864ad4..51857daff 100644--- a/tests/adapters/test_xml_adapter.py+++ b/tests/adapters/test_xml_adapter.py@@ -302,6 +302,158 @@ def test_xml_adapter_full_prompt():assert messages[1]["content"] == expected_user+def test_xml_adapter_format_exact_messages_for_simple_signature():+ class StringSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField()++ messages = dspy.XMLAdapter().format(+ StringSignature,+ demos=[],+ inputs={"question": "why did a chicken cross the kitchen?"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answer` (str):+All interactions will be structured in the following way, with the appropriate values filled in.++<question>+{question}+</question>++<answer>+{answer}+</answer>+In adhering to this structure, your objective is:\x20+ Given the fields `question`, produce the fields `answer`.""",+ },+ {+ "role": "user",+ "content": """<question>+why did a chicken cross the kitchen?+</question>++Respond with the corresponding output fields wrapped in XML tags `<answer>`.""",+ },+ ]+++def test_xml_adapter_format_exact_messages_for_two_input_signature():+ class StringSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.InputField()+ judgement: str = dspy.OutputField()++ messages = dspy.XMLAdapter().format(+ StringSignature,+ demos=[],+ inputs={"question": "why did a chicken cross the kitchen?", "answer": "To get to the other side!"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):\x20+2. `answer` (str):+Your output fields are:+1. `judgement` (str):+All interactions will be structured in the following way, with the appropriate values filled in.++<question>+{question}+</question>++<answer>+{answer}+</answer>++<judgement>+{judgement}+</judgement>+In adhering to this structure, your objective is:\x20+ Given the fields `question`, `answer`, produce the fields `judgement`.""",+ },+ {+ "role": "user",+ "content": """<question>+why did a chicken cross the kitchen?+</question>++<answer>+To get to the other side!+</answer>++Respond with the corresponding output fields wrapped in XML tags `<judgement>`.""",+ },+ ]+++def test_xml_adapter_format_exact_messages_with_demo_and_typed_output():+ class MultiAnswer(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField()+ score: float = dspy.OutputField()++ messages = dspy.XMLAdapter().format(+ MultiAnswer,+ demos=[{"question": "Q1", "answer": "A1", "score": 0.9}],+ inputs={"question": "Q2"},+ )++ assert messages == [+ {+ "role": "system",+ "content": """Your input fields are:+1. `question` (str):+Your output fields are:+1. `answer` (str):\x20+2. `score` (float):+All interactions will be structured in the following way, with the appropriate values filled in.++<question>+{question}+</question>++<answer>+{answer}+</answer>++<score>+{score} # note: the value you produce must be a single float value+</score>+In adhering to this structure, your objective is:\x20+ Given the fields `question`, produce the fields `answer`, `score`.""",+ },+ {"role": "user", "content": """<question>+Q1+</question>"""},+ {+ "role": "assistant",+ "content": """<answer>+A1+</answer>++<score>+0.9+</score>""",+ },+ {+ "role": "user",+ "content": """<question>+Q2+</question>++Respond with the corresponding output fields wrapped in XML tags `<answer>`, then `<score>`.""",+ },+ ]++def test_format_system_message():class MySignature(dspy.Signature):"""Answer the question with multiple answers and scores"""