DSPy: The framework for programming—not prompting—language models
Python
Allow custom type to be streamed and use native response (#8778)
TomuHirata10fb57994730
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.pyindex f714b338b..701f84cce 100644--- a/dspy/adapters/base.py+++ b/dspy/adapters/base.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, get_originimport json_repairimport litellm-from dspy.adapters.types import History+from dspy.adapters.types import History, Typefrom dspy.adapters.types.base_type import split_message_content_for_custom_typesfrom dspy.adapters.types.tool import Tool, ToolCallsfrom dspy.experimental import Citations@@ -16,11 +16,13 @@ logger = logging.getLogger(__name__)if TYPE_CHECKING:from dspy.clients.lm import LM+_DEFAULT_NATIVE_RESPONSE_TYPES = [Citations]class Adapter:- def __init__(self, callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False):+ def __init__(self, callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False, native_response_types: list[type[Type]] | None = None):self.callbacks = callbacks or []self.use_native_function_calling = use_native_function_calling+ self.native_response_types = native_response_types or _DEFAULT_NATIVE_RESPONSE_TYPESdef __init_subclass__(cls, **kwargs) -> None:super().__init_subclass__(**kwargs)@@ -64,9 +66,10 @@ class Adapter:return signature_for_native_function_calling- citation_output_field_name = self._get_citation_output_field_name(signature)- if citation_output_field_name:- signature = signature.delete(citation_output_field_name)+ # Handle custom types that use native response+ for name, field in signature.output_fields.items():+ if isinstance(field.annotation, type) and issubclass(field.annotation, Type) and field.annotation in self.native_response_types:+ signature = signature.delete(name)return signature@@ -75,23 +78,21 @@ class Adapter:processed_signature: type[Signature],original_signature: type[Signature],outputs: list[dict[str, Any]],+ lm: "LM",) -> list[dict[str, Any]]:values = []tool_call_output_field_name = self._get_tool_call_output_field_name(original_signature)- citation_output_field_name = self._get_citation_output_field_name(original_signature)for output in outputs:output_logprobs = Nonetool_calls = None- citations = Nonetext = outputif isinstance(output, dict):text = output["text"]output_logprobs = output.get("logprobs")tool_calls = output.get("tool_calls")- citations = output.get("citations")if text:value = self.parse(processed_signature, text)@@ -114,9 +115,10 @@ class Adapter:]value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls)- if citations and citation_output_field_name:- citations_obj = Citations.from_dict_list(citations)- value[citation_output_field_name] = citations_obj+ # Parse custom types that does not rely on the adapter parsing+ for name, field in original_signature.output_fields.items():+ if isinstance(field.annotation, type) and issubclass(field.annotation, Type) and field.annotation in self.native_response_types:+ value[name] = field.annotation.parse_lm_response(output)if output_logprobs:value["logprobs"] = output_logprobs@@ -137,7 +139,7 @@ class Adapter:inputs = self.format(processed_signature, demos, inputs)outputs = lm(messages=inputs, **lm_kwargs)- return self._call_postprocess(processed_signature, signature, outputs)+ return self._call_postprocess(processed_signature, signature, outputs, lm)async def acall(self,@@ -151,7 +153,7 @@ class Adapter:inputs = self.format(processed_signature, demos, inputs)outputs = await lm.acall(messages=inputs, **lm_kwargs)- return self._call_postprocess(processed_signature, signature, outputs)+ return self._call_postprocess(processed_signature, signature, outputs, lm)def format(self,@@ -402,12 +404,6 @@ class Adapter:return namereturn None- def _get_citation_output_field_name(self, signature: type[Signature]) -> str | None:- """Find the Citations output field in the signature."""- for name, field in signature.output_fields.items():- if field.annotation == Citations:- return name- return Nonedef format_conversation_history(self,diff --git a/dspy/adapters/types/base_type.py b/dspy/adapters/types/base_type.pyindex 8f82b87b7..04e2794ce 100644--- a/dspy/adapters/types/base_type.py+++ b/dspy/adapters/types/base_type.py@@ -1,9 +1,10 @@import jsonimport re-from typing import Any, get_args, get_origin+from typing import Any, Optional, get_args, get_originimport json_repairimport pydantic+from litellm import ModelResponseStreamCUSTOM_TYPE_START_IDENTIFIER = "<<CUSTOM-TYPE-START-IDENTIFIER>>"CUSTOM_TYPE_END_IDENTIFIER = "<<CUSTOM-TYPE-END-IDENTIFIER>>"@@ -69,6 +70,36 @@ class Type(pydantic.BaseModel):)return formatted+ @classmethod+ def is_streamable(cls) -> bool:+ """Whether the custom type is streamable."""+ return False++ @classmethod+ def parse_stream_chunk(cls, chunk: ModelResponseStream) -> Optional["Type"]:+ """+ Parse a stream chunk into the custom type.++ Args:+ chunk: A stream chunk.++ Returns:+ A custom type object or None if the chunk is not for this custom type.+ """+ return None+++ @classmethod+ def parse_lm_response(cls, response: str | dict[str, Any]) -> Optional["Type"]:+ """Parse a LM response into the custom type.++ Args:+ response: A LM response.++ Returns:+ A custom type object.+ """+ return Nonedef split_message_content_for_custom_types(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:"""Split user message content into a list of content blocks.diff --git a/dspy/adapters/types/citation.py b/dspy/adapters/types/citation.pyindex b3c613c07..82e55f5a2 100644--- a/dspy/adapters/types/citation.py+++ b/dspy/adapters/types/citation.py@@ -1,4 +1,4 @@-from typing import Any+from typing import Any, Optionalimport pydantic@@ -166,3 +166,51 @@ class Citations(Type):def __getitem__(self, index):"""Allow indexing into citations."""return self.citations[index]++ @classmethod+ def is_streamable(cls) -> bool:+ """Whether the Citations type is streamable."""+ return True++ @classmethod+ def parse_stream_chunk(cls, chunk) -> Optional["Citations"]:+ """+ Parse a stream chunk into Citations.++ Args:+ chunk: A stream chunk from the LM.++ Returns:+ A Citations object if the chunk contains citation data, None otherwise.+ """+ try:+ # Check if the chunk has citation data in provider_specific_fields+ if hasattr(chunk, "choices") and chunk.choices:+ delta = chunk.choices[0].delta+ if hasattr(delta, "provider_specific_fields") and delta.provider_specific_fields:+ citation_data = delta.provider_specific_fields.get("citation")+ if citation_data:+ return cls.from_dict_list([citation_data])+ except Exception:+ pass+ return None+++ @classmethod+ def parse_lm_response(cls, response: str | dict[str, Any]) -> Optional["Citations"]:+ """Parse a LM response into Citations.++ Args:+ response: A LM response that may contain citation data.++ Returns:+ A Citations object if citation data is found, None otherwise.+ """+ if isinstance(response, dict):+ # Check if the response contains citations in the expected format+ if "citations" in response:+ citations_data = response["citations"]+ if isinstance(citations_data, list):+ return cls.from_dict_list(citations_data)++ return Nonediff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex e30759351..7b7835d4c 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -7,7 +7,7 @@ from litellm import ModelResponseStreamfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapter-from dspy.adapters.types.citation import Citations+from dspy.adapters.types import Typefrom dspy.adapters.xml_adapter import XMLAdapterfrom dspy.dsp.utils.settings import settingsfrom dspy.streaming.messages import StreamResponse@@ -102,18 +102,15 @@ class StreamListener:except Exception:return- # Handle anthropic citations. see https://docs.litellm.ai/docs/providers/anthropic#beta-citations-api- try:- if self._signature_field_is_citation_type():- if chunk_citation := chunk.choices[0].delta.provider_specific_fields.get("citation", None):- return StreamResponse(- self.predict_name,- self.signature_field_name,- Citations.from_dict_list([chunk_citation]),- is_last_chunk=False,- )- except Exception:- pass+ # Handle custom streamable types+ if self._output_type and issubclass(self._output_type, Type) and self._output_type.is_streamable():+ if parsed_chunk := self._output_type.parse_stream_chunk(chunk):+ return StreamResponse(+ self.predict_name,+ self.signature_field_name,+ parsed_chunk,+ is_last_chunk=self.stream_end,+ )if chunk_message and start_identifier in chunk_message:# If the cache is hit, the chunk_message could be the full response. When it happens we can@@ -217,10 +214,13 @@ class StreamListener:f"{', '.join([a.__name__ for a in ADAPTER_SUPPORT_STREAMING])}")- def _signature_field_is_citation_type(self) -> bool:- """Check if the signature field is a citations field."""- from dspy.predict import Predict- return isinstance(self.predict, Predict) and getattr(self.predict.signature.output_fields.get(self.signature_field_name, None), "annotation", None) == Citations+ @property+ def _output_type(self) -> type | None:+ try:+ return self.predict.signature.output_fields[self.signature_field_name].annotation+ except Exception:+ return None+def find_predictor_for_stream_listeners(program: "Module", stream_listeners: list[StreamListener]):@@ -249,10 +249,10 @@ def find_predictor_for_stream_listeners(program: "Module", stream_listeners: lis"predictor to use for streaming. Please specify the predictor to listen to.")- if field_info.annotation not in [str, Citations]:+ if not _is_streamable(field_info.annotation):raise ValueError(- f"Stream listener can only be applied to string or Citations output field, but your field {field_name} is of "- f"type {field_info.annotation}."+ f"Stream listener can only be applied to string or subclass of `dspy.Type` that has `is_streamable() == True`, "+ f"but your field {field_name} is of type {field_info.annotation}.")field_name_to_named_predictor[field_name] = (name, predictor)@@ -271,3 +271,12 @@ def find_predictor_for_stream_listeners(program: "Module", stream_listeners: lislistener.predict_name, listener.predict = field_name_to_named_predictor[listener.signature_field_name]predict_id_to_listener[id(listener.predict)].append(listener)return predict_id_to_listener++def _is_streamable(field_type: type | None) -> bool:+ if field_type is None:+ return False+ if field_type is str:+ return True+ if issubclass(field_type, Type):+ return field_type.is_streamable()+ return Falsediff --git a/tests/adapters/test_citation.py b/tests/adapters/test_citation.pyindex 8d2881b07..d1fdc8d08 100644--- a/tests/adapters/test_citation.py+++ b/tests/adapters/test_citation.py@@ -135,7 +135,7 @@ def test_citations_postprocessing():answer: str = dspy.OutputField()citations: Citations = dspy.OutputField()- adapter = ChatAdapter()+ adapter = ChatAdapter(native_response_types=[Citations])outputs = [{"text": "[[ ## answer ## ]]\nThe answer is blue.\n\n[[ ## citations ## ]]\n[]",@@ -154,7 +154,8 @@ def test_citations_postprocessing():result = adapter._call_postprocess(CitationSignature.delete("citations"),CitationSignature,- outputs+ outputs,+ dspy.LM(model="claude-3-5-sonnet-20241022"))assert len(result) == 1diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 134320d61..b7afa7dc4 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -9,6 +9,7 @@ from asyncer import syncifyfrom litellm.types.utils import Delta, ModelResponseStream, StreamingChoicesimport dspy+from dspy.adapters.types import Typefrom dspy.experimental import Citations, Documentfrom dspy.streaming import StatusMessage, StatusMessageProvider, streaming_response@@ -877,6 +878,58 @@ async def test_streaming_allows_custom_chunk_types():assert isinstance(all_chunks[1], dspy.Prediction)+async def test_streaming_allows_custom_streamable_type():+ class CustomType(Type):+ message: str++ @classmethod+ def is_streamable(cls) -> bool:+ return True++ @classmethod+ def parse_stream_chunk(cls, chunk):+ return CustomType(message=chunk.choices[0].delta.content)++ @classmethod+ def parse_lm_response(cls, response: dict) -> "CustomType":+ return CustomType(message=response.split("\n\n")[0])++ class CustomSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: CustomType = dspy.OutputField()++ program = dspy.streamify(+ dspy.Predict(CustomSignature),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ ],+ )++ async def stream(*args, **kwargs):+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="Hello"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="World"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" completed"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])+++ with mock.patch("litellm.acompletion", side_effect=stream):+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter(native_response_types=[CustomType])):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)+ elif isinstance(value, dspy.Prediction):+ assert isinstance(value.answer, CustomType)+ assert value.answer.message == "HelloWorld"++ assert all(isinstance(chunk.chunk, CustomType) for chunk in all_chunks)++@pytest.mark.anyioasync def test_streaming_with_citations():class AnswerWithSources(dspy.Signature):@@ -936,7 +989,7 @@ async def test_streaming_with_citations():# Create test documentsdocs = [Document(data="Water boils at 100°C at standard pressure.", title="Physics Facts")]- with dspy.context(lm=dspy.LM("anthropic/claude-3-5-sonnet-20241022", cache=False)):+ with dspy.context(lm=dspy.LM("anthropic/claude-3-5-sonnet-20241022", cache=False), adapter=dspy.ChatAdapter(native_response_types=[Citations])):output = program(documents=docs, question="What temperature does water boil?")citation_chunks = []final_prediction = None