Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add exact adapter format message tests (#9791)

Maxime Rivest52efe77814e1
diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py
index 86c662b3b..cfa546660 100644
--- a/tests/adapters/test_chat_adapter.py
+++ b/tests/adapters/test_chat_adapter.py
@@ -99,6 +99,104 @@ async def test_chat_adapter_async_call():
assert result == [{"answer": "Paris"}]
+def test_chat_adapter_format_exact_messages_for_simple_signature():
+ class QA(dspy.Signature):
+ """Answer the question."""
+
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+
+ messages = dspy.ChatAdapter().format(QA, [], {"question": "What is the capital of France?"})
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answer` (str):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+[[ ## question ## ]]
+{question}
+
+[[ ## answer ## ]]
+{answer}
+
+[[ ## completed ## ]]
+In adhering to this structure, your objective is:\x20
+ Answer the question.""",
+ },
+ {
+ "role": "user",
+ "content": """[[ ## question ## ]]
+What is the capital of France?
+
+Respond with the corresponding output fields, starting with the field `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.""",
+ },
+ ]
+
+
+def test_chat_adapter_format_exact_messages_with_demo_and_typed_outputs():
+ class MultiAnswer(dspy.Signature):
+ """Answer the question with multiple answers and scores"""
+
+ question: str = dspy.InputField()
+ answers: list[str] = dspy.OutputField()
+ scores: list[float] = dspy.OutputField()
+
+ messages = dspy.ChatAdapter().format(
+ MultiAnswer,
+ demos=[{"question": "Q1", "answers": ["A1", "A2"], "scores": [0.1, 0.9]}],
+ inputs={"question": "Q2"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answers` (list[str]):\x20
+2. `scores` (list[float]):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+[[ ## question ## ]]
+{question}
+
+[[ ## answers ## ]]
+{answers} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "string"}}
+
+[[ ## scores ## ]]
+{scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}}
+
+[[ ## completed ## ]]
+In adhering to this structure, your objective is:\x20
+ Answer the question with multiple answers and scores""",
+ },
+ {"role": "user", "content": """[[ ## question ## ]]
+Q1"""},
+ {
+ "role": "assistant",
+ "content": """[[ ## answers ## ]]
+["A1", "A2"]
+
+[[ ## scores ## ]]
+[0.1, 0.9]
+
+[[ ## completed ## ]]
+""",
+ },
+ {
+ "role": "user",
+ "content": """[[ ## question ## ]]
+Q2
+
+Respond with the corresponding output fields, starting with the field `[[ ## answers ## ]]` (must be formatted as a valid Python list[str]), then `[[ ## scores ## ]]` (must be formatted as a valid Python list[float]), and then ending with the marker for `[[ ## completed ## ]]`.""",
+ },
+ ]
+
+
def test_chat_adapter_with_pydantic_models():
"""
This test verifies that ChatAdapter can handle different input and output field types, both basic and nested.
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index f376831d9..13ca0ea81 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -9,6 +9,146 @@ from openai.types.responses import ResponseOutputMessage
import dspy
+def test_json_adapter_format_exact_messages_for_simple_signature():
+ class StringSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+
+ messages = dspy.JSONAdapter().format(
+ StringSignature,
+ demos=[],
+ inputs={"question": "What is the capital of France?"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answer` (str):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+Inputs will have the following structure:
+
+[[ ## question ## ]]
+{question}
+
+Outputs will be a JSON object with the following fields.
+
+{
+ "answer": "{answer}"
+}
+In adhering to this structure, your objective is:\x20
+ Given the fields `question`, produce the fields `answer`.""",
+ },
+ {
+ "role": "user",
+ "content": """[[ ## question ## ]]
+What is the capital of France?
+
+Respond with a JSON object in the following order of fields: `answer`.""",
+ },
+ ]
+
+
+def test_json_adapter_format_exact_messages_with_demo_and_typed_output():
+ class MultiAnswer(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+ confidence: float = dspy.OutputField()
+
+ messages = dspy.JSONAdapter().format(
+ MultiAnswer,
+ demos=[{"question": "Q1", "answer": "A1", "confidence": 0.9}],
+ inputs={"question": "Q2"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answer` (str):\x20
+2. `confidence` (float):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+Inputs will have the following structure:
+
+[[ ## question ## ]]
+{question}
+
+Outputs will be a JSON object with the following fields.
+
+{
+ "answer": "{answer}",
+ "confidence": "{confidence} # note: the value you produce must be a single float value"
+}
+In adhering to this structure, your objective is:\x20
+ Given the fields `question`, produce the fields `answer`, `confidence`.""",
+ },
+ {"role": "user", "content": """[[ ## question ## ]]
+Q1"""},
+ {
+ "role": "assistant",
+ "content": """{
+ "answer": "A1",
+ "confidence": 0.9
+}""",
+ },
+ {
+ "role": "user",
+ "content": """[[ ## question ## ]]
+Q2
+
+Respond with a JSON object in the following order of fields: `answer`, then `confidence` (must be formatted as a valid Python float).""",
+ },
+ ]
+
+
+def test_json_adapter_format_exact_messages_with_described_and_bool_outputs():
+ class TestSignature(dspy.Signature):
+ input1: str = dspy.InputField()
+ output1: str = dspy.OutputField(desc="String output field")
+ output2: bool = dspy.OutputField()
+
+ messages = dspy.JSONAdapter().format(TestSignature, [], {"input1": "Test input"})
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `input1` (str):
+Your output fields are:
+1. `output1` (str): String output field
+2. `output2` (bool):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+Inputs will have the following structure:
+
+[[ ## input1 ## ]]
+{input1}
+
+Outputs will be a JSON object with the following fields.
+
+{
+ "output1": "{output1}",
+ "output2": "{output2} # note: the value you produce must be True or False"
+}
+In adhering to this structure, your objective is:\x20
+ Given the fields `input1`, produce the fields `output1`, `output2`.""",
+ },
+ {
+ "role": "user",
+ "content": """[[ ## input1 ## ]]
+Test input
+
+Respond with a JSON object in the following order of fields: `output1`, then `output2` (must be formatted as a valid Python bool).""",
+ },
+ ]
+
+
def test_json_adapter_passes_structured_output_when_supported_by_model():
class OutputField3(pydantic.BaseModel):
subfield1: int = pydantic.Field(description="Int subfield 1", ge=0, le=10)
diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.py
index ded864ad4..51857daff 100644
--- a/tests/adapters/test_xml_adapter.py
+++ b/tests/adapters/test_xml_adapter.py
@@ -302,6 +302,158 @@ def test_xml_adapter_full_prompt():
assert messages[1]["content"] == expected_user
+def test_xml_adapter_format_exact_messages_for_simple_signature():
+ class StringSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+
+ messages = dspy.XMLAdapter().format(
+ StringSignature,
+ demos=[],
+ inputs={"question": "why did a chicken cross the kitchen?"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answer` (str):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+<question>
+{question}
+</question>
+
+<answer>
+{answer}
+</answer>
+In adhering to this structure, your objective is:\x20
+ Given the fields `question`, produce the fields `answer`.""",
+ },
+ {
+ "role": "user",
+ "content": """<question>
+why did a chicken cross the kitchen?
+</question>
+
+Respond with the corresponding output fields wrapped in XML tags `<answer>`.""",
+ },
+ ]
+
+
+def test_xml_adapter_format_exact_messages_for_two_input_signature():
+ class StringSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.InputField()
+ judgement: str = dspy.OutputField()
+
+ messages = dspy.XMLAdapter().format(
+ StringSignature,
+ demos=[],
+ inputs={"question": "why did a chicken cross the kitchen?", "answer": "To get to the other side!"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):\x20
+2. `answer` (str):
+Your output fields are:
+1. `judgement` (str):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+<question>
+{question}
+</question>
+
+<answer>
+{answer}
+</answer>
+
+<judgement>
+{judgement}
+</judgement>
+In adhering to this structure, your objective is:\x20
+ Given the fields `question`, `answer`, produce the fields `judgement`.""",
+ },
+ {
+ "role": "user",
+ "content": """<question>
+why did a chicken cross the kitchen?
+</question>
+
+<answer>
+To get to the other side!
+</answer>
+
+Respond with the corresponding output fields wrapped in XML tags `<judgement>`.""",
+ },
+ ]
+
+
+def test_xml_adapter_format_exact_messages_with_demo_and_typed_output():
+ class MultiAnswer(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+ score: float = dspy.OutputField()
+
+ messages = dspy.XMLAdapter().format(
+ MultiAnswer,
+ demos=[{"question": "Q1", "answer": "A1", "score": 0.9}],
+ inputs={"question": "Q2"},
+ )
+
+ assert messages == [
+ {
+ "role": "system",
+ "content": """Your input fields are:
+1. `question` (str):
+Your output fields are:
+1. `answer` (str):\x20
+2. `score` (float):
+All interactions will be structured in the following way, with the appropriate values filled in.
+
+<question>
+{question}
+</question>
+
+<answer>
+{answer}
+</answer>
+
+<score>
+{score} # note: the value you produce must be a single float value
+</score>
+In adhering to this structure, your objective is:\x20
+ Given the fields `question`, produce the fields `answer`, `score`.""",
+ },
+ {"role": "user", "content": """<question>
+Q1
+</question>"""},
+ {
+ "role": "assistant",
+ "content": """<answer>
+A1
+</answer>
+
+<score>
+0.9
+</score>""",
+ },
+ {
+ "role": "user",
+ "content": """<question>
+Q2
+</question>
+
+Respond with the corresponding output fields wrapped in XML tags `<answer>`, then `<score>`.""",
+ },
+ ]
+
+
def test_format_system_message():
class MySignature(dspy.Signature):
"""Answer the question with multiple answers and scores"""