DSPy: The framework for programming—not prompting—language models
Python
Add token streaming support for XMLAdapter (#8478)
Chen Qian982e20f76fb4
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex 1cedeeaed..91489e207 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -7,12 +7,15 @@ from litellm import ModelResponseStreamfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapter+from dspy.adapters.xml_adapter import XMLAdapterfrom dspy.dsp.utils.settings import settingsfrom dspy.streaming.messages import StreamResponseif TYPE_CHECKING:from dspy.primitives.module import Module+ADAPTER_SUPPORT_STREAMING = [ChatAdapter, XMLAdapter, JSONAdapter]+class StreamListener:"""Class that listens to the stream to capture the streeaming of a specific output field of a predictor."""@@ -45,11 +48,23 @@ class StreamListener:self.cache_hit = Falseself.allow_reuse = allow_reuse- self.json_adapter_start_identifier = f'"{self.signature_field_name}":'- self.json_adapter_end_identifier = re.compile(r"\w*\"(,|\s*})")-- self.chat_adapter_start_identifier = f"[[ ## {self.signature_field_name} ## ]]"- self.chat_adapter_end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")+ self.adapter_identifiers = {+ "ChatAdapter": {+ "start_identifier": f"[[ ## {self.signature_field_name} ## ]]",+ "end_identifier": re.compile(r"\[\[ ## (\w+) ## \]\]"),+ "start_indicator": "[",+ },+ "JSONAdapter": {+ "start_identifier": f'"{self.signature_field_name}":',+ "end_identifier": re.compile(r"\w*\"(,|\s*})"),+ "start_indicator": '"',+ },+ "XMLAdapter": {+ "start_identifier": f"<{self.signature_field_name}>",+ "end_identifier": re.compile(rf"</{self.signature_field_name}>"),+ "start_indicator": "<",+ },+ }def _buffered_message_end_with_start_identifier(self, concat_message: str, start_identifier: str) -> str:for i in range(len(concat_message)):@@ -58,21 +73,15 @@ class StreamListener:return Falsedef receive(self, chunk: ModelResponseStream):- if isinstance(settings.adapter, JSONAdapter):- start_identifier = self.json_adapter_start_identifier- end_identifier = self.json_adapter_end_identifier-- start_indicator = '"'- elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:- start_identifier = self.chat_adapter_start_identifier- end_identifier = self.chat_adapter_end_identifier-- start_indicator = "["- else:+ adapter_name = settings.adapter.__class__.__name__ if settings.adapter else "ChatAdapter"+ if adapter_name not in self.adapter_identifiers:raise ValueError(- f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "- "JSONAdapter for streaming purposes."+ f"Unsupported adapter for streaming: {adapter_name}, please use one of the following adapters: "+ f"{', '.join([a.__name__ for a in ADAPTER_SUPPORT_STREAMING])}")+ start_identifier = self.adapter_identifiers[adapter_name]["start_identifier"]+ end_identifier = self.adapter_identifiers[adapter_name]["end_identifier"]+ start_indicator = self.adapter_identifiers[adapter_name]["start_indicator"]if self.stream_end:if self.allow_reuse:@@ -175,13 +184,18 @@ class StreamListener:else:boundary_index = len(last_tokens)return last_tokens[:boundary_index]+ elif isinstance(settings.adapter, XMLAdapter):+ boundary_index = last_tokens.find(f"</{self.signature_field_name}>")+ if boundary_index == -1:+ boundary_index = len(last_tokens)+ return last_tokens[:boundary_index]elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:boundary_index = last_tokens.find("[[")return last_tokens[:boundary_index]else:raise ValueError(- f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "- "JSONAdapter for streaming purposes."+ f"Unsupported adapter for streaming: {settings.adapter}, please use one of the following adapters: "+ f"{', '.join([a.__name__ for a in ADAPTER_SUPPORT_STREAMING])}")diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 56c5426bb..fe67bd6da 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -387,7 +387,7 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():async def completion_side_effect(*args, **kwargs):return stream_generators.pop(0)() # return new async generator instance- with mock.patch("litellm.acompletion", side_effect=completion_side_effect) as mock_completion:+ with mock.patch("litellm.acompletion", side_effect=completion_side_effect):program = dspy.streamify(MyProgram(),stream_listeners=[@@ -483,7 +483,7 @@ async def test_stream_listener_returns_correct_chunk_json_adapter():with mock.patch("litellm.acompletion", new_callable=AsyncMock, side_effect=[gpt_4o_mini_stream_1(), gpt_4o_mini_stream_2()]- ) as mock_completion:+ ):program = dspy.streamify(MyProgram(),stream_listeners=[@@ -762,3 +762,78 @@ async def test_stream_listener_allow_reuse():concat_message = "".join([chunk.chunk for chunk in all_chunks])# The listener functions twice.assert concat_message == "To get to the other side!To get to the other side!"++async def test_stream_listener_returns_correct_chunk_xml_adapter():+ class MyProgram(dspy.Module):+ def __init__(self):+ super().__init__()+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question,answer->judgement")++ def forward(self, question, **kwargs):+ answer = self.predict1(question=question, **kwargs).answer+ judgement = self.predict2(question=question, answer=answer, **kwargs)+ return judgement++ async def xml_stream_1(*args, **kwargs):+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="To"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" get"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" to"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" other"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" side"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="!"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="/answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="completed"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])++ async def xml_stream_2(*args, **kwargs):+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="judgement"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="The"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" is"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" humorous"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="."))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="/judgement"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="<"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="completed"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=">"))])++ stream_generators = [xml_stream_1, xml_stream_2]++ async def completion_side_effect(*args, **kwargs):+ return stream_generators.pop(0)()++ with mock.patch("litellm.acompletion", side_effect=completion_side_effect):+ program = dspy.streamify(+ MyProgram(),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ )+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.XMLAdapter()):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"+ assert all_chunks[0].chunk == "To get to the other side!"++ assert all_chunks[1].predict_name == "predict2"+ assert all_chunks[1].signature_field_name == "judgement"+ assert all_chunks[1].chunk == "The answer is humorous."