DSPy: The framework for programming—not prompting—language models
Python
Introduce two step adapter (#8011)
TomuHirata32095d378b13
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 70e7de708..3e440344a 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -8,7 +8,7 @@ import dspy.retrieversfrom dspy.evaluate import Evaluate # isort: skipfrom dspy.clients import * # isort: skip-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, Image, History # isort: skip+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, TwoStepAdapter, Image, History # isort: skipfrom dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_loggingfrom dspy.utils.asyncify import asyncifyfrom dspy.utils.saving import loaddiff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.pyindex c592ac6ba..f6a128a7e 100644--- a/dspy/adapters/__init__.py+++ b/dspy/adapters/__init__.py@@ -2,6 +2,7 @@ from dspy.adapters.base import Adapterfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapterfrom dspy.adapters.types import Image, History+from dspy.adapters.two_step_adapter import TwoStepAdapter__all__ = ["Adapter",@@ -9,4 +10,5 @@ __all__ = ["JSONAdapter","Image","History",+ "TwoStepAdapter",]diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.pynew file mode 100644index 000000000..1004b0ff7--- /dev/null+++ b/dspy/adapters/two_step_adapter.py@@ -0,0 +1,170 @@+from typing import Any, Type++from dspy.signatures.signature import Signature+from dspy.adapters.base import Adapter+from dspy.adapters.chat_adapter import ChatAdapter+from dspy.adapters.utils import get_field_description_string+from dspy.signatures.field import InputField+from dspy.signatures.signature import make_signature+from dspy.clients import LM+++"""+NOTE/TODO/FIMXE:++The main issue below is that the second step's signature is entirely created on the fly and is invoked with a chat+adapter explicitly constructed with no demonstrations. This means that it cannot "learn" or get optimized.+"""+++class TwoStepAdapter(Adapter):+ """+ A two-stage adapter that:+ 1. Uses a simpler, more natural prompt for the main LM+ 2. Uses a smaller LM with chat adapter to extract structured data from the response of main LM+ This adapter uses a common __call__ logic defined in base Adapter class.+ This class is particularly useful when interacting with reasoning models as the main LM since reasoning models+ are known to struggle with structured outputs.++ Example:+ ```+ import dspy+ lm = dspy.LM(model="openai/o3-mini", max_tokens=10000, temperature = 1.0)+ adapter = dspy.TwoStepAdapter(dspy.LM("openai/gpt-4o-mini"))+ dspy.configure(lm=lm, adapter=adapter)+ program = dspy.ChainOfThought("question->answer")+ result = program("What is the capital of France?")+ print(result)+ ```+ """++ def __init__(self, extraction_model: LM):+ if not isinstance(extraction_model, LM):+ raise ValueError("extraction_model must be an instance of LM")+ self.extraction_model = extraction_model++ def format(+ self, signature: Type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any]+ ) -> list[dict[str, Any]]:+ """+ Format a prompt for the first stage with the main LM.+ This no specific structure is required for the main LM, we customize the format method+ instead of format_field_description or format_field_structure.++ Args:+ signature: The signature of the original task+ demos: A list of demo examples+ inputs: The current input++ Returns:+ A list of messages to be passed to the main LM.+ """+ messages = []++ # Create a task description for the main LM+ task_description = self.format_task_description(signature)+ messages.append({"role": "system", "content": task_description})++ messages.extend(self.format_demos(signature, demos))++ # Format the current input+ messages.append({"role": "user", "content": self.format_user_message_content(signature, inputs)})++ return messages++ def parse(self, signature: Signature, completion: str) -> dict[str, Any]:+ """+ Use a smaller LM (extraction_model) with chat adapter to extract structured data+ from the raw completion text of the main LM.++ Args:+ signature: The signature of the original task+ completion: The completion from the main LM++ Returns:+ A dictionary containing the extracted structured data.+ """+ # The signature is supposed to be "text -> {original output fields}"+ extractor_signature = self._create_extractor_signature(signature)++ try:+ # Call the smaller LM to extract structured data from the raw completion text with ChatAdapter+ parsed_result = ChatAdapter()(+ lm=self.extraction_model,+ lm_kwargs={},+ signature=extractor_signature,+ demos=[],+ inputs={"text": completion},+ )+ return parsed_result[0]++ except Exception as e:+ raise ValueError(f"Failed to parse response from the original completion: {completion}") from e++ def format_task_description(self, signature: Signature) -> str:+ """Create a description of the task based on the signature"""+ parts = []++ parts.append("You are a helpful assistant that can solve tasks based on user input.")+ parts.append("As input, you will be provided with:\n" + get_field_description_string(signature.input_fields))+ parts.append("Your outputs must contain:\n" + get_field_description_string(signature.output_fields))+ parts.append("You should lay out your outputs in detail so that your answer can be understood by another agent")++ if signature.instructions:+ parts.append(f"Specific instructions: {signature.instructions}")++ return "\n".join(parts)++ def format_user_message_content(+ self,+ signature: Type[Signature],+ inputs: dict[str, Any],+ prefix: str = "",+ suffix: str = "",+ ) -> str:+ parts = [prefix]++ for name in signature.input_fields.keys():+ if name in inputs:+ parts.append(f"{name}: {inputs.get(name, '')}")++ parts.append(suffix)+ return "\n\n".join(parts).strip()++ def format_assistant_message_content(+ self,+ signature: Type[Signature],+ outputs: dict[str, Any],+ missing_field_message: str = None,+ ) -> str:+ parts = []++ for name in signature.output_fields.keys():+ if name in outputs:+ parts.append(f"{name}: {outputs.get(name, missing_field_message)}")++ return "\n\n".join(parts).strip()++ def _create_extractor_signature(+ self,+ original_signature: Type[Signature],+ ) -> Type[Signature]:+ """Create a new signature containing a new 'text' input field and all output fields.++ Args:+ original_signature: The original signature to extract output fields from++ Returns:+ A new Signature type with a text input field and all output fields+ """+ # Create new fields dict with 'text' input field and all output fields+ new_fields = {+ "text": (str, InputField()),+ **{name: (field.annotation, field) for name, field in original_signature.output_fields.items()},+ }++ outputs_str = ", ".join([f"`{field}`" for field in original_signature.output_fields])+ instructions = f"The input is a text that should contain all the necessary information to produce the fields {outputs_str}. \+ Your job is to extract the fields from the text verbatim. Extract precisely the appropriate value (content) for each field."++ return make_signature(new_fields, instructions)diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 4906dc93d..f878ea640 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -90,8 +90,8 @@ class LM(BaseLM):if model_pattern:# Handle OpenAI reasoning models (o1, o3)assert (- max_tokens >= 5000 and temperature == 1.0- ), "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"+ max_tokens >= 20_000 and temperature == 1.0+ ), "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 20_000 to `dspy.LM(...)`"self.kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)else:self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.pynew file mode 100644index 000000000..304d4651e--- /dev/null+++ b/tests/adapters/test_two_step_adapter.py@@ -0,0 +1,113 @@+from unittest import mock+import pytest++import dspy+++def test_two_step_adapter_call():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField(desc="The math question to solve")+ solution: str = dspy.OutputField(desc="Step by step solution")+ answer: float = dspy.OutputField(desc="The final numerical answer")++ program = dspy.Predict(TestSignature)++ mock_main_lm = mock.MagicMock(spec=dspy.LM)+ mock_main_lm.return_value = ["text from main LM"]+ mock_main_lm.kwargs = {"temperature": 1.0}++ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)+ mock_extraction_lm.return_value = [+ """+[[ ## solution ## ]] result+[[ ## answer ## ]] 12+[[ ## completed ## ]]+"""+ ]+ mock_extraction_lm.kwargs = {"temperature": 1.0}+ mock_extraction_lm.model = "openai/gpt-4o"++ dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))++ result = program(question="What is 5 + 7?")++ assert result.answer == 12++ # main LM call+ mock_main_lm.assert_called_once()+ _, call_kwargs = mock_main_lm.call_args+ assert len(call_kwargs["messages"]) == 2++ # assert first message+ assert call_kwargs["messages"][0]["role"] == "system"+ content = call_kwargs["messages"][0]["content"]+ assert "1. `question` (str)" in content+ assert "1. `solution` (str)" in content+ assert "2. `answer` (float)" in content++ # assert second message+ assert call_kwargs["messages"][1]["role"] == "user"+ content = call_kwargs["messages"][1]["content"]+ assert "question:" in content.lower()+ assert "What is 5 + 7?" in content++ # extraction LM call+ mock_extraction_lm.assert_called_once()+ _, call_kwargs = mock_extraction_lm.call_args+ assert len(call_kwargs["messages"]) == 2++ # assert first message+ assert call_kwargs["messages"][0]["role"] == "system"+ content = call_kwargs["messages"][0]["content"]+ assert "`text` (str)" in content+ assert "`solution` (str)" in content+ assert "`answer` (float)" in content++ # assert second message+ assert call_kwargs["messages"][1]["role"] == "user"+ content = call_kwargs["messages"][1]["content"]+ assert "text from main LM" in content+++def test_two_step_adapter_parse():+ class ComplexSignature(dspy.Signature):+ input_text: str = dspy.InputField()+ tags: list[str] = dspy.OutputField(desc="List of relevant tags")+ confidence: float = dspy.OutputField(desc="Confidence score")++ first_response = "main LM response"++ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)+ mock_extraction_lm.return_value = ["""+ {+ "tags": ["AI", "deep learning", "neural networks"],+ "confidence": 0.87+ }+ """]+ mock_extraction_lm.kwargs = {"temperature": 1.0}+ mock_extraction_lm.model = "openai/gpt-4o"+ adapter = dspy.TwoStepAdapter(mock_extraction_lm)+ dspy.configure(adapter=adapter, lm=mock_extraction_lm)++ result = adapter.parse(ComplexSignature, first_response)++ assert result["tags"] == ["AI", "deep learning", "neural networks"]+ assert result["confidence"] == 0.87+++def test_two_step_adapter_parse_errors():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField()++ first_response = "main LM response"++ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)+ mock_extraction_lm.return_value = ["invalid response"]+ mock_extraction_lm.kwargs = {"temperature": 1.0}+ mock_extraction_lm.model = "openai/gpt-4o"++ adapter = dspy.TwoStepAdapter(mock_extraction_lm)++ with pytest.raises(ValueError, match="Failed to parse response"):+ adapter.parse(TestSignature, first_response)diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 87b2f2757..a68e20d72 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -199,12 +199,12 @@ def test_reasoning_model_token_parameter():lm = dspy.LM(model=model_name,temperature=1.0 if is_reasoning_model else 0.7,- max_tokens=5000 if is_reasoning_model else 1000,+ max_tokens=20_000 if is_reasoning_model else 1000,)if is_reasoning_model:assert "max_completion_tokens" in lm.kwargsassert "max_tokens" not in lm.kwargs- assert lm.kwargs["max_completion_tokens"] == 5000+ assert lm.kwargs["max_completion_tokens"] == 20_000else:assert "max_completion_tokens" not in lm.kwargsassert "max_tokens" in lm.kwargs@@ -217,17 +217,17 @@ def test_reasoning_model_requirements():dspy.LM(model="openai/o1",temperature=0.7, # Should be 1.0- max_tokens=1000, # Should be >= 5000+ max_tokens=1000, # Should be >= 20_000)- assert "reasoning models require passing temperature=1.0 and max_tokens >= 5000" in str(exc_info.value)+ assert "reasoning models require passing temperature=1.0 and max_tokens >= 20_000" in str(exc_info.value)# Should pass with correct parameterslm = dspy.LM(model="openai/o1",temperature=1.0,- max_tokens=5000,+ max_tokens=20_000,)- assert lm.kwargs["max_completion_tokens"] == 5000+ assert lm.kwargs["max_completion_tokens"] == 20_000def test_dump_state():diff --git a/tests/reliability/conftest.py b/tests/reliability/conftest.pyindex 9aaef2f28..8dfcf7e2f 100644--- a/tests/reliability/conftest.py+++ b/tests/reliability/conftest.py@@ -21,6 +21,7 @@ MODEL_LIST = ["llama-3.1-70b-instruct","llama-3.1-8b-instruct","llama-3.2-3b-instruct",+ "deepseek-r1",]diff --git a/tests/reliability/reliability_conf.yaml b/tests/reliability/reliability_conf.yamlindex ef33cd94e..1a18428e8 100644--- a/tests/reliability/reliability_conf.yaml+++ b/tests/reliability/reliability_conf.yaml@@ -73,3 +73,9 @@ model_list:# model: "<litellm_provider>/<litellm_model_name>"# api_key: "api key"# api_base: "<api_base>"+ - model_name: "deepseek-r1"+ litellm_params:+ # model: "<litellm_provider>/<litellm_model_name>"+ # api_key: "api key"+ # max_tokens: 10000+diff --git a/tests/reliability/test_pydantic_models.py b/tests/reliability/test_pydantic_models.pyindex 9e62ce517..1853fc885 100644--- a/tests/reliability/test_pydantic_models.py+++ b/tests/reliability/test_pydantic_models.py@@ -90,18 +90,18 @@ def test_entity_extraction_with_multiple_primitive_outputs():assert_program_output_correct(program_input=description,program_output=extracted_entity.entity_hu,- grading_guidelines="The translation of the text into English should be equivalent to 'coffee'",+ grading_guidelines="The translation of the extracted entity into English should be equivalent to 'coffee'",)assert_program_output_correct(program_input=description,- program_output=extracted_entity.entity_hu,- grading_guidelines="The text should be equivalent to 'coffee'",+ program_output=extracted_entity.entity_en,+ grading_guidelines="The extracted entity should be equivalent to 'coffee'",)assert_program_output_correct(program_input=description,program_output=extracted_entity.categories,grading_guidelines=(- "The text should contain English language categories that apply to the word 'coffee'."+ "The extracted entity should be associated with English language categories that apply to the word 'coffee'."" The categories should be separated by the character '|'."),)