Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Introduce two step adapter (#8011)

TomuHirata32095d378b13
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 70e7de708..3e440344a 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -8,7 +8,7 @@ import dspy.retrievers
from dspy.evaluate import Evaluate # isort: skip
from dspy.clients import * # isort: skip
-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, Image, History # isort: skip
+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, TwoStepAdapter, Image, History # isort: skip
from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging
from dspy.utils.asyncify import asyncify
from dspy.utils.saving import load
diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py
index c592ac6ba..f6a128a7e 100644
--- a/dspy/adapters/__init__.py
+++ b/dspy/adapters/__init__.py
@@ -2,6 +2,7 @@ from dspy.adapters.base import Adapter
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.json_adapter import JSONAdapter
from dspy.adapters.types import Image, History
+from dspy.adapters.two_step_adapter import TwoStepAdapter
__all__ = [
"Adapter",
@@ -9,4 +10,5 @@ __all__ = [
"JSONAdapter",
"Image",
"History",
+ "TwoStepAdapter",
]
diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.py
new file mode 100644
index 000000000..1004b0ff7
--- /dev/null
+++ b/dspy/adapters/two_step_adapter.py
@@ -0,0 +1,170 @@
+from typing import Any, Type
+
+from dspy.signatures.signature import Signature
+from dspy.adapters.base import Adapter
+from dspy.adapters.chat_adapter import ChatAdapter
+from dspy.adapters.utils import get_field_description_string
+from dspy.signatures.field import InputField
+from dspy.signatures.signature import make_signature
+from dspy.clients import LM
+
+
+"""
+NOTE/TODO/FIMXE:
+
+The main issue below is that the second step's signature is entirely created on the fly and is invoked with a chat
+adapter explicitly constructed with no demonstrations. This means that it cannot "learn" or get optimized.
+"""
+
+
+class TwoStepAdapter(Adapter):
+ """
+ A two-stage adapter that:
+ 1. Uses a simpler, more natural prompt for the main LM
+ 2. Uses a smaller LM with chat adapter to extract structured data from the response of main LM
+ This adapter uses a common __call__ logic defined in base Adapter class.
+ This class is particularly useful when interacting with reasoning models as the main LM since reasoning models
+ are known to struggle with structured outputs.
+
+ Example:
+ ```
+ import dspy
+ lm = dspy.LM(model="openai/o3-mini", max_tokens=10000, temperature = 1.0)
+ adapter = dspy.TwoStepAdapter(dspy.LM("openai/gpt-4o-mini"))
+ dspy.configure(lm=lm, adapter=adapter)
+ program = dspy.ChainOfThought("question->answer")
+ result = program("What is the capital of France?")
+ print(result)
+ ```
+ """
+
+ def __init__(self, extraction_model: LM):
+ if not isinstance(extraction_model, LM):
+ raise ValueError("extraction_model must be an instance of LM")
+ self.extraction_model = extraction_model
+
+ def format(
+ self, signature: Type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any]
+ ) -> list[dict[str, Any]]:
+ """
+ Format a prompt for the first stage with the main LM.
+ This no specific structure is required for the main LM, we customize the format method
+ instead of format_field_description or format_field_structure.
+
+ Args:
+ signature: The signature of the original task
+ demos: A list of demo examples
+ inputs: The current input
+
+ Returns:
+ A list of messages to be passed to the main LM.
+ """
+ messages = []
+
+ # Create a task description for the main LM
+ task_description = self.format_task_description(signature)
+ messages.append({"role": "system", "content": task_description})
+
+ messages.extend(self.format_demos(signature, demos))
+
+ # Format the current input
+ messages.append({"role": "user", "content": self.format_user_message_content(signature, inputs)})
+
+ return messages
+
+ def parse(self, signature: Signature, completion: str) -> dict[str, Any]:
+ """
+ Use a smaller LM (extraction_model) with chat adapter to extract structured data
+ from the raw completion text of the main LM.
+
+ Args:
+ signature: The signature of the original task
+ completion: The completion from the main LM
+
+ Returns:
+ A dictionary containing the extracted structured data.
+ """
+ # The signature is supposed to be "text -> {original output fields}"
+ extractor_signature = self._create_extractor_signature(signature)
+
+ try:
+ # Call the smaller LM to extract structured data from the raw completion text with ChatAdapter
+ parsed_result = ChatAdapter()(
+ lm=self.extraction_model,
+ lm_kwargs={},
+ signature=extractor_signature,
+ demos=[],
+ inputs={"text": completion},
+ )
+ return parsed_result[0]
+
+ except Exception as e:
+ raise ValueError(f"Failed to parse response from the original completion: {completion}") from e
+
+ def format_task_description(self, signature: Signature) -> str:
+ """Create a description of the task based on the signature"""
+ parts = []
+
+ parts.append("You are a helpful assistant that can solve tasks based on user input.")
+ parts.append("As input, you will be provided with:\n" + get_field_description_string(signature.input_fields))
+ parts.append("Your outputs must contain:\n" + get_field_description_string(signature.output_fields))
+ parts.append("You should lay out your outputs in detail so that your answer can be understood by another agent")
+
+ if signature.instructions:
+ parts.append(f"Specific instructions: {signature.instructions}")
+
+ return "\n".join(parts)
+
+ def format_user_message_content(
+ self,
+ signature: Type[Signature],
+ inputs: dict[str, Any],
+ prefix: str = "",
+ suffix: str = "",
+ ) -> str:
+ parts = [prefix]
+
+ for name in signature.input_fields.keys():
+ if name in inputs:
+ parts.append(f"{name}: {inputs.get(name, '')}")
+
+ parts.append(suffix)
+ return "\n\n".join(parts).strip()
+
+ def format_assistant_message_content(
+ self,
+ signature: Type[Signature],
+ outputs: dict[str, Any],
+ missing_field_message: str = None,
+ ) -> str:
+ parts = []
+
+ for name in signature.output_fields.keys():
+ if name in outputs:
+ parts.append(f"{name}: {outputs.get(name, missing_field_message)}")
+
+ return "\n\n".join(parts).strip()
+
+ def _create_extractor_signature(
+ self,
+ original_signature: Type[Signature],
+ ) -> Type[Signature]:
+ """Create a new signature containing a new 'text' input field and all output fields.
+
+ Args:
+ original_signature: The original signature to extract output fields from
+
+ Returns:
+ A new Signature type with a text input field and all output fields
+ """
+ # Create new fields dict with 'text' input field and all output fields
+ new_fields = {
+ "text": (str, InputField()),
+ **{name: (field.annotation, field) for name, field in original_signature.output_fields.items()},
+ }
+
+ outputs_str = ", ".join([f"`{field}`" for field in original_signature.output_fields])
+ instructions = f"The input is a text that should contain all the necessary information to produce the fields {outputs_str}. \
+ Your job is to extract the fields from the text verbatim. Extract precisely the appropriate value (content) for each field."
+
+ return make_signature(new_fields, instructions)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 4906dc93d..f878ea640 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -90,8 +90,8 @@ class LM(BaseLM):
if model_pattern:
# Handle OpenAI reasoning models (o1, o3)
assert (
- max_tokens >= 5000 and temperature == 1.0
- ), "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"
+ max_tokens >= 20_000 and temperature == 1.0
+ ), "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 20_000 to `dspy.LM(...)`"
self.kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)
else:
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py
new file mode 100644
index 000000000..304d4651e
--- /dev/null
+++ b/tests/adapters/test_two_step_adapter.py
@@ -0,0 +1,113 @@
+from unittest import mock
+import pytest
+
+import dspy
+
+
+def test_two_step_adapter_call():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField(desc="The math question to solve")
+ solution: str = dspy.OutputField(desc="Step by step solution")
+ answer: float = dspy.OutputField(desc="The final numerical answer")
+
+ program = dspy.Predict(TestSignature)
+
+ mock_main_lm = mock.MagicMock(spec=dspy.LM)
+ mock_main_lm.return_value = ["text from main LM"]
+ mock_main_lm.kwargs = {"temperature": 1.0}
+
+ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)
+ mock_extraction_lm.return_value = [
+ """
+[[ ## solution ## ]] result
+[[ ## answer ## ]] 12
+[[ ## completed ## ]]
+"""
+ ]
+ mock_extraction_lm.kwargs = {"temperature": 1.0}
+ mock_extraction_lm.model = "openai/gpt-4o"
+
+ dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))
+
+ result = program(question="What is 5 + 7?")
+
+ assert result.answer == 12
+
+ # main LM call
+ mock_main_lm.assert_called_once()
+ _, call_kwargs = mock_main_lm.call_args
+ assert len(call_kwargs["messages"]) == 2
+
+ # assert first message
+ assert call_kwargs["messages"][0]["role"] == "system"
+ content = call_kwargs["messages"][0]["content"]
+ assert "1. `question` (str)" in content
+ assert "1. `solution` (str)" in content
+ assert "2. `answer` (float)" in content
+
+ # assert second message
+ assert call_kwargs["messages"][1]["role"] == "user"
+ content = call_kwargs["messages"][1]["content"]
+ assert "question:" in content.lower()
+ assert "What is 5 + 7?" in content
+
+ # extraction LM call
+ mock_extraction_lm.assert_called_once()
+ _, call_kwargs = mock_extraction_lm.call_args
+ assert len(call_kwargs["messages"]) == 2
+
+ # assert first message
+ assert call_kwargs["messages"][0]["role"] == "system"
+ content = call_kwargs["messages"][0]["content"]
+ assert "`text` (str)" in content
+ assert "`solution` (str)" in content
+ assert "`answer` (float)" in content
+
+ # assert second message
+ assert call_kwargs["messages"][1]["role"] == "user"
+ content = call_kwargs["messages"][1]["content"]
+ assert "text from main LM" in content
+
+
+def test_two_step_adapter_parse():
+ class ComplexSignature(dspy.Signature):
+ input_text: str = dspy.InputField()
+ tags: list[str] = dspy.OutputField(desc="List of relevant tags")
+ confidence: float = dspy.OutputField(desc="Confidence score")
+
+ first_response = "main LM response"
+
+ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)
+ mock_extraction_lm.return_value = ["""
+ {
+ "tags": ["AI", "deep learning", "neural networks"],
+ "confidence": 0.87
+ }
+ """]
+ mock_extraction_lm.kwargs = {"temperature": 1.0}
+ mock_extraction_lm.model = "openai/gpt-4o"
+ adapter = dspy.TwoStepAdapter(mock_extraction_lm)
+ dspy.configure(adapter=adapter, lm=mock_extraction_lm)
+
+ result = adapter.parse(ComplexSignature, first_response)
+
+ assert result["tags"] == ["AI", "deep learning", "neural networks"]
+ assert result["confidence"] == 0.87
+
+
+def test_two_step_adapter_parse_errors():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+
+ first_response = "main LM response"
+
+ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)
+ mock_extraction_lm.return_value = ["invalid response"]
+ mock_extraction_lm.kwargs = {"temperature": 1.0}
+ mock_extraction_lm.model = "openai/gpt-4o"
+
+ adapter = dspy.TwoStepAdapter(mock_extraction_lm)
+
+ with pytest.raises(ValueError, match="Failed to parse response"):
+ adapter.parse(TestSignature, first_response)
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 87b2f2757..a68e20d72 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -199,12 +199,12 @@ def test_reasoning_model_token_parameter():
lm = dspy.LM(
model=model_name,
temperature=1.0 if is_reasoning_model else 0.7,
- max_tokens=5000 if is_reasoning_model else 1000,
+ max_tokens=20_000 if is_reasoning_model else 1000,
)
if is_reasoning_model:
assert "max_completion_tokens" in lm.kwargs
assert "max_tokens" not in lm.kwargs
- assert lm.kwargs["max_completion_tokens"] == 5000
+ assert lm.kwargs["max_completion_tokens"] == 20_000
else:
assert "max_completion_tokens" not in lm.kwargs
assert "max_tokens" in lm.kwargs
@@ -217,17 +217,17 @@ def test_reasoning_model_requirements():
dspy.LM(
model="openai/o1",
temperature=0.7, # Should be 1.0
- max_tokens=1000, # Should be >= 5000
+ max_tokens=1000, # Should be >= 20_000
)
- assert "reasoning models require passing temperature=1.0 and max_tokens >= 5000" in str(exc_info.value)
+ assert "reasoning models require passing temperature=1.0 and max_tokens >= 20_000" in str(exc_info.value)
# Should pass with correct parameters
lm = dspy.LM(
model="openai/o1",
temperature=1.0,
- max_tokens=5000,
+ max_tokens=20_000,
)
- assert lm.kwargs["max_completion_tokens"] == 5000
+ assert lm.kwargs["max_completion_tokens"] == 20_000
def test_dump_state():
diff --git a/tests/reliability/conftest.py b/tests/reliability/conftest.py
index 9aaef2f28..8dfcf7e2f 100644
--- a/tests/reliability/conftest.py
+++ b/tests/reliability/conftest.py
@@ -21,6 +21,7 @@ MODEL_LIST = [
"llama-3.1-70b-instruct",
"llama-3.1-8b-instruct",
"llama-3.2-3b-instruct",
+ "deepseek-r1",
]
diff --git a/tests/reliability/reliability_conf.yaml b/tests/reliability/reliability_conf.yaml
index ef33cd94e..1a18428e8 100644
--- a/tests/reliability/reliability_conf.yaml
+++ b/tests/reliability/reliability_conf.yaml
@@ -73,3 +73,9 @@ model_list:
# model: "<litellm_provider>/<litellm_model_name>"
# api_key: "api key"
# api_base: "<api_base>"
+ - model_name: "deepseek-r1"
+ litellm_params:
+ # model: "<litellm_provider>/<litellm_model_name>"
+ # api_key: "api key"
+ # max_tokens: 10000
+
diff --git a/tests/reliability/test_pydantic_models.py b/tests/reliability/test_pydantic_models.py
index 9e62ce517..1853fc885 100644
--- a/tests/reliability/test_pydantic_models.py
+++ b/tests/reliability/test_pydantic_models.py
@@ -90,18 +90,18 @@ def test_entity_extraction_with_multiple_primitive_outputs():
assert_program_output_correct(
program_input=description,
program_output=extracted_entity.entity_hu,
- grading_guidelines="The translation of the text into English should be equivalent to 'coffee'",
+ grading_guidelines="The translation of the extracted entity into English should be equivalent to 'coffee'",
)
assert_program_output_correct(
program_input=description,
- program_output=extracted_entity.entity_hu,
- grading_guidelines="The text should be equivalent to 'coffee'",
+ program_output=extracted_entity.entity_en,
+ grading_guidelines="The extracted entity should be equivalent to 'coffee'",
)
assert_program_output_correct(
program_input=description,
program_output=extracted_entity.categories,
grading_guidelines=(
- "The text should contain English language categories that apply to the word 'coffee'."
+ "The extracted entity should be associated with English language categories that apply to the word 'coffee'."
" The categories should be separated by the character '|'."
),
)