DSPy: The framework for programming—not prompting—language models
Python
Support models that don't split stream chunks in tokens (#8235)
Chen Qian5d31cd102fe2
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex c95086b03..8d9ad0682 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -36,12 +36,18 @@ class StreamListener:self.stream_end = Falseself.cache_hit = False- self.json_adapter_start_identifier = f'{{"{self.signature_field_name}":"'- self.json_adapter_end_identifier = re.compile(r"\w*\",\w*")+ self.json_adapter_start_identifier = f'"{self.signature_field_name}":'+ self.json_adapter_end_identifier = re.compile(r"\w*\"(,|\s*})")self.chat_adapter_start_identifier = f"[[ ## {self.signature_field_name} ## ]]"self.chat_adapter_end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")+ def _buffered_message_end_with_start_identifier(self, concat_message: str, start_identifier: str) -> str:+ for i in range(len(concat_message)):+ if start_identifier.startswith(concat_message[len(concat_message) - i - 1 :]):+ return True+ return False+def receive(self, chunk: ModelResponseStream):if isinstance(settings.adapter, JSONAdapter):start_identifier = self.json_adapter_start_identifier@@ -52,7 +58,7 @@ class StreamListener:start_identifier = self.chat_adapter_start_identifierend_identifier = self.chat_adapter_end_identifier- start_indicator = "[["+ start_indicator = "["else:raise ValueError(f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "@@ -71,13 +77,18 @@ class StreamListener:if chunk_message and start_identifier in chunk_message:# If the cache is hit, the chunk_message could be the full response. When it happens we can- # directly end the stream listening.- self.cache_hit = True- self.stream_start = True- self.stream_end = True- return+ # directly end the stream listening. In some models like gemini, each stream chunk can be multiple+ # tokens, so it's posible that response only has one chunk, we also fall back to this logic.+ message_after_start_identifier = chunk_message[+ chunk_message.find(start_identifier) + len(start_identifier) :+ ]+ if re.search(end_identifier, message_after_start_identifier):+ self.cache_hit = True+ self.stream_start = True+ self.stream_end = True+ return- if len(self.field_start_queue) == 0 and start_indicator in chunk_message:+ if len(self.field_start_queue) == 0 and not self.stream_start and start_indicator in chunk_message:# We look for the pattern of start_identifier, i.e., "[[ ## {self.signature_field_name} ## ]]" for# ChatAdapter to identify the start of the stream of our target field. Once the start_indicator, i.e., "[["# for ChatAdapter, is found, we start checking the next tokens@@ -89,18 +100,28 @@ class StreamListener:# tokens no longer match our expected identifier.self.field_start_queue.append(chunk_message)concat_message = "".join(self.field_start_queue).strip()- start_token_index = concat_message.find(start_indicator)- concat_message = concat_message[start_token_index:]- if start_identifier == concat_message:++ if start_identifier in concat_message:# We have a full identifier, we can start the stream.self.stream_start = True- elif start_identifier.startswith(concat_message):- # The concanated tokens still match our expected identifier, we keep listening.+ self.field_start_queue = []+ # Keep the part after the start_identifier from the concat_message, we need to write it to the buffer.+ value_start_index = concat_message.find(start_identifier) + len(start_identifier)+ chunk_message = concat_message[value_start_index:].lstrip()+ if isinstance(settings.adapter, JSONAdapter) and chunk_message.startswith('"'):+ # For JSONAdapter, we need to remove the leading ". We cannot do this with the start_identifier+ # because there could be a few splitters between ':' and '"', e.g., '"name": "value"'.+ chunk_message = chunk_message[1:]++ elif self._buffered_message_end_with_start_identifier(concat_message, start_identifier):+ # If the buffered message ends with part of the start_identifier, we can start the stream.returnelse:# Doesn't match the expected identifier, reset the queue.self.field_start_queue = []- elif self.stream_start:+ return++ if self.stream_start:# The stream is started, we keep returning the token until we see the start of the next field.token = Noneself.field_end_queue.put(chunk_message)@@ -130,7 +151,11 @@ class StreamListener:last_tokens = "".join(self.field_end_queue.queue)self.field_end_queue = Queue()if isinstance(settings.adapter, JSONAdapter):- boundary_index = last_tokens.find('",')+ match = re.search(r'",|"\s*}', last_tokens)+ if match:+ boundary_index = match.start()+ else:+ boundary_index = len(last_tokens)return last_tokens[:boundary_index]elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:boundary_index = last_tokens.find("[[")diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 9363f23cb..23dead5e7 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -4,6 +4,9 @@ import pytestimport dspyfrom dspy.streaming import StatusMessage, StatusMessageProvider, streaming_response+from unittest import mock+from unittest.mock import AsyncMock+from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta@pytest.mark.anyio@@ -287,3 +290,333 @@ def test_sync_status_streaming():assert len(status_messages) == 2assert status_messages[0].message == "Calling tool generate_question..."assert status_messages[1].message == "Tool calling finished! Querying the LLM with tool calling results..."+++async def test_stream_listener_returns_correct_chunk_chat_adapter():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question, answer->judgement")++ def forward(self, question, **kwargs):+ answer = self.predict1(question=question, **kwargs).answer+ judgement = self.predict2(question=question, answer=answer, **kwargs)+ return judgement++ async def gpt_4o_mini_stream_1(*args, **kwargs):+ # Recorded streaming from openai/gpt-4o-mini+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[["))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]\n\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="To"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" get"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" to"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" other"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" side"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" of"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" dinner"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plate"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="!"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" completed"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])++ async def gpt_4o_mini_stream_2():+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" judgement"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]\n\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="The"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" is"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" humorous"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plays"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" on"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" classic"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" joke"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" format"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="."))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" completed"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))++ stream_generators = [gpt_4o_mini_stream_1, gpt_4o_mini_stream_2]++ async def completion_side_effect(*args, **kwargs):+ return stream_generators.pop(0)() # return new async generator instance++ with mock.patch("litellm.acompletion", side_effect=completion_side_effect) as mock_completion:+ program = dspy.streamify(+ MyProgram(),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ )+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"++ assert all_chunks[0].chunk == "To"+ assert all_chunks[1].chunk == " get"+ assert all_chunks[2].chunk == " to"+ assert all_chunks[3].chunk == " the"+ assert all_chunks[4].chunk == " other"+ assert all_chunks[5].chunk == " side of the dinner plate!"++ # Start processing the second listened field.+ assert all_chunks[6].predict_name == "predict2"+ assert all_chunks[6].signature_field_name == "judgement"+ assert all_chunks[6].chunk == "The"+ assert all_chunks[7].chunk == " answer"+ assert all_chunks[8].chunk == " is"+ assert all_chunks[9].chunk == " humorous"+ assert all_chunks[10].chunk == " and"+ assert all_chunks[11].chunk == " plays"+++async def test_stream_listener_returns_correct_chunk_json_adapter():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question,answer->judgement")++ def forward(self, question, **kwargs):+ answer = self.predict1(question=question, **kwargs).answer+ judgement = self.predict2(question=question, answer=answer, **kwargs)+ return judgement++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())++ async def gpt_4o_mini_stream_1(*args, **kwargs):+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='":'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="To"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" get"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" to"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" other"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" side"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" of"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" frying"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" pan"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='!"'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="}\n"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])++ async def gpt_4o_mini_stream_2(*args, **kwargs):+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="jud"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="gement"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='":"'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="The"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" is"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" humorous"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plays"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" on"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" very"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" funny"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" classic"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" joke"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" format"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='."'))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="}"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])++ with mock.patch(+ "litellm.acompletion", new_callable=AsyncMock, side_effect=[gpt_4o_mini_stream_1(), gpt_4o_mini_stream_2()]+ ) as mock_completion:+ program = dspy.streamify(+ MyProgram(),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ )+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"++ assert all_chunks[0].chunk == "To"+ assert all_chunks[1].chunk == " get to the other side of the frying pan!"++ # Start processing the second listened field.+ assert all_chunks[2].predict_name == "predict2"+ assert all_chunks[2].signature_field_name == "judgement"+ assert all_chunks[2].chunk == "The"+ assert all_chunks[3].chunk == " answer"+ assert all_chunks[4].chunk == " is"+ assert all_chunks[5].chunk == " humorous"+ assert all_chunks[6].chunk == " and"+ assert all_chunks[7].chunk == " plays on the very funny and classic joke format."+++async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_stream():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question,answer->judgement")++ def forward(self, question, **kwargs):+ answer = self.predict1(question=question, **kwargs).answer+ judgement = self.predict2(question=question, answer=answer, **kwargs)+ return judgement++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter())++ async def gemini_stream_1(*args, **kwargs):+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content=" answer ## ]]"))])+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content="To get to the other side."))]+ )+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))]+ )++ async def gemini_stream_2(*args, **kwargs):+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ## judgement ## ]]\n\n"))]+ )+ yield ModelResponseStream(+ model="gemini",+ choices=[+ StreamingChoices(+ delta=Delta(+ content=(+ "The answer provides the standard punchline for this classic joke format, adapted to the "+ "specific location mentioned in the question. It is the expected and appropriate response."+ )+ )+ )+ ],+ )+ yield ModelResponseStream(+ model="gemini",+ choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))],+ )+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])++ with mock.patch("litellm.acompletion", new_callable=AsyncMock, side_effect=[gemini_stream_1(), gemini_stream_2()]):+ program = dspy.streamify(+ MyProgram(),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ )+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"+ assert all_chunks[0].chunk == "To get to the other side."++ assert all_chunks[1].predict_name == "predict2"+ assert all_chunks[1].signature_field_name == "judgement"+ assert all_chunks[1].chunk == (+ "The answer provides the standard punchline for this classic joke format, adapted to the specific location "+ "mentioned in the question. It is the expected and appropriate response."+ )+++async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_stream():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question,answer->judgement")++ def forward(self, question, **kwargs):+ answer = self.predict1(question=question, **kwargs).answer+ judgement = self.predict2(question=question, answer=answer, **kwargs)+ return judgement++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())++ async def gemini_stream_1(*args, **kwargs):+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' "answer": "To get to'))]+ )+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' the other side... of the cutting board!"'))]+ )+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])++ async def gemini_stream_2(*args, **kwargs):+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])+ yield ModelResponseStream(+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' "judgement": "The'))]+ )+ yield ModelResponseStream(+ model="gemini",+ choices=[+ StreamingChoices(+ delta=Delta(+ content=' answer provides a humorous and relevant punchline to the classic joke setup."'+ )+ )+ ],+ )+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])++ with mock.patch("litellm.acompletion", new_callable=AsyncMock, side_effect=[gemini_stream_1(), gemini_stream_2()]):+ program = dspy.streamify(+ MyProgram(),+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ )+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"+ assert all_chunks[0].chunk == "To get to the other side... of the cutting board!"++ assert all_chunks[1].predict_name == "predict2"+ assert all_chunks[1].signature_field_name == "judgement"+ assert all_chunks[1].chunk == "The answer provides a humorous and relevant punchline to the classic joke setup."