Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Support models that don't split stream chunks in tokens (#8235)

Chen Qian5d31cd102fe2
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
index c95086b03..8d9ad0682 100644
--- a/dspy/streaming/streaming_listener.py
+++ b/dspy/streaming/streaming_listener.py
@@ -36,12 +36,18 @@ class StreamListener:
self.stream_end = False
self.cache_hit = False
- self.json_adapter_start_identifier = f'{{"{self.signature_field_name}":"'
- self.json_adapter_end_identifier = re.compile(r"\w*\",\w*")
+ self.json_adapter_start_identifier = f'"{self.signature_field_name}":'
+ self.json_adapter_end_identifier = re.compile(r"\w*\"(,|\s*})")
self.chat_adapter_start_identifier = f"[[ ## {self.signature_field_name} ## ]]"
self.chat_adapter_end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")
+ def _buffered_message_end_with_start_identifier(self, concat_message: str, start_identifier: str) -> str:
+ for i in range(len(concat_message)):
+ if start_identifier.startswith(concat_message[len(concat_message) - i - 1 :]):
+ return True
+ return False
+
def receive(self, chunk: ModelResponseStream):
if isinstance(settings.adapter, JSONAdapter):
start_identifier = self.json_adapter_start_identifier
@@ -52,7 +58,7 @@ class StreamListener:
start_identifier = self.chat_adapter_start_identifier
end_identifier = self.chat_adapter_end_identifier
- start_indicator = "[["
+ start_indicator = "["
else:
raise ValueError(
f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "
@@ -71,13 +77,18 @@ class StreamListener:
if chunk_message and start_identifier in chunk_message:
# If the cache is hit, the chunk_message could be the full response. When it happens we can
- # directly end the stream listening.
- self.cache_hit = True
- self.stream_start = True
- self.stream_end = True
- return
+ # directly end the stream listening. In some models like gemini, each stream chunk can be multiple
+ # tokens, so it's posible that response only has one chunk, we also fall back to this logic.
+ message_after_start_identifier = chunk_message[
+ chunk_message.find(start_identifier) + len(start_identifier) :
+ ]
+ if re.search(end_identifier, message_after_start_identifier):
+ self.cache_hit = True
+ self.stream_start = True
+ self.stream_end = True
+ return
- if len(self.field_start_queue) == 0 and start_indicator in chunk_message:
+ if len(self.field_start_queue) == 0 and not self.stream_start and start_indicator in chunk_message:
# We look for the pattern of start_identifier, i.e., "[[ ## {self.signature_field_name} ## ]]" for
# ChatAdapter to identify the start of the stream of our target field. Once the start_indicator, i.e., "[["
# for ChatAdapter, is found, we start checking the next tokens
@@ -89,18 +100,28 @@ class StreamListener:
# tokens no longer match our expected identifier.
self.field_start_queue.append(chunk_message)
concat_message = "".join(self.field_start_queue).strip()
- start_token_index = concat_message.find(start_indicator)
- concat_message = concat_message[start_token_index:]
- if start_identifier == concat_message:
+
+ if start_identifier in concat_message:
# We have a full identifier, we can start the stream.
self.stream_start = True
- elif start_identifier.startswith(concat_message):
- # The concanated tokens still match our expected identifier, we keep listening.
+ self.field_start_queue = []
+ # Keep the part after the start_identifier from the concat_message, we need to write it to the buffer.
+ value_start_index = concat_message.find(start_identifier) + len(start_identifier)
+ chunk_message = concat_message[value_start_index:].lstrip()
+ if isinstance(settings.adapter, JSONAdapter) and chunk_message.startswith('"'):
+ # For JSONAdapter, we need to remove the leading ". We cannot do this with the start_identifier
+ # because there could be a few splitters between ':' and '"', e.g., '"name": "value"'.
+ chunk_message = chunk_message[1:]
+
+ elif self._buffered_message_end_with_start_identifier(concat_message, start_identifier):
+ # If the buffered message ends with part of the start_identifier, we can start the stream.
return
else:
# Doesn't match the expected identifier, reset the queue.
self.field_start_queue = []
- elif self.stream_start:
+ return
+
+ if self.stream_start:
# The stream is started, we keep returning the token until we see the start of the next field.
token = None
self.field_end_queue.put(chunk_message)
@@ -130,7 +151,11 @@ class StreamListener:
last_tokens = "".join(self.field_end_queue.queue)
self.field_end_queue = Queue()
if isinstance(settings.adapter, JSONAdapter):
- boundary_index = last_tokens.find('",')
+ match = re.search(r'",|"\s*}', last_tokens)
+ if match:
+ boundary_index = match.start()
+ else:
+ boundary_index = len(last_tokens)
return last_tokens[:boundary_index]
elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:
boundary_index = last_tokens.find("[[")
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 9363f23cb..23dead5e7 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -4,6 +4,9 @@ import pytest
import dspy
from dspy.streaming import StatusMessage, StatusMessageProvider, streaming_response
+from unittest import mock
+from unittest.mock import AsyncMock
+from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta
@pytest.mark.anyio
@@ -287,3 +290,333 @@ def test_sync_status_streaming():
assert len(status_messages) == 2
assert status_messages[0].message == "Calling tool generate_question..."
assert status_messages[1].message == "Tool calling finished! Querying the LLM with tool calling results..."
+
+
+async def test_stream_listener_returns_correct_chunk_chat_adapter():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.predict1 = dspy.Predict("question->answer")
+ self.predict2 = dspy.Predict("question, answer->judgement")
+
+ def forward(self, question, **kwargs):
+ answer = self.predict1(question=question, **kwargs).answer
+ judgement = self.predict2(question=question, answer=answer, **kwargs)
+ return judgement
+
+ async def gpt_4o_mini_stream_1(*args, **kwargs):
+ # Recorded streaming from openai/gpt-4o-mini
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[["))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]\n\n"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="To"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" get"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" to"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" other"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" side"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" of"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" dinner"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plate"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="!"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" completed"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])
+
+ async def gpt_4o_mini_stream_2():
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" judgement"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]\n\n"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="The"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" is"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" humorous"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plays"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" on"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" classic"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" joke"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" format"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="."))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" completed"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
+
+ stream_generators = [gpt_4o_mini_stream_1, gpt_4o_mini_stream_2]
+
+ async def completion_side_effect(*args, **kwargs):
+ return stream_generators.pop(0)() # return new async generator instance
+
+ with mock.patch("litellm.acompletion", side_effect=completion_side_effect) as mock_completion:
+ program = dspy.streamify(
+ MyProgram(),
+ stream_listeners=[
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="judgement"),
+ ],
+ )
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+
+ assert all_chunks[0].predict_name == "predict1"
+ assert all_chunks[0].signature_field_name == "answer"
+
+ assert all_chunks[0].chunk == "To"
+ assert all_chunks[1].chunk == " get"
+ assert all_chunks[2].chunk == " to"
+ assert all_chunks[3].chunk == " the"
+ assert all_chunks[4].chunk == " other"
+ assert all_chunks[5].chunk == " side of the dinner plate!"
+
+ # Start processing the second listened field.
+ assert all_chunks[6].predict_name == "predict2"
+ assert all_chunks[6].signature_field_name == "judgement"
+ assert all_chunks[6].chunk == "The"
+ assert all_chunks[7].chunk == " answer"
+ assert all_chunks[8].chunk == " is"
+ assert all_chunks[9].chunk == " humorous"
+ assert all_chunks[10].chunk == " and"
+ assert all_chunks[11].chunk == " plays"
+
+
+async def test_stream_listener_returns_correct_chunk_json_adapter():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.predict1 = dspy.Predict("question->answer")
+ self.predict2 = dspy.Predict("question,answer->judgement")
+
+ def forward(self, question, **kwargs):
+ answer = self.predict1(question=question, **kwargs).answer
+ judgement = self.predict2(question=question, answer=answer, **kwargs)
+ return judgement
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
+
+ async def gpt_4o_mini_stream_1(*args, **kwargs):
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="answer"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='":'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="To"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" get"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" to"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" other"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" side"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" of"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" frying"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" pan"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='!"'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="}\n"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+
+ async def gpt_4o_mini_stream_2(*args, **kwargs):
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="jud"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="gement"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='":"'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="The"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" answer"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" is"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" humorous"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" plays"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" on"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" the"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" very"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" funny"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" and"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" classic"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" joke"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" format"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='."'))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="}"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="None"))])
+
+ with mock.patch(
+ "litellm.acompletion", new_callable=AsyncMock, side_effect=[gpt_4o_mini_stream_1(), gpt_4o_mini_stream_2()]
+ ) as mock_completion:
+ program = dspy.streamify(
+ MyProgram(),
+ stream_listeners=[
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="judgement"),
+ ],
+ )
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+
+ assert all_chunks[0].predict_name == "predict1"
+ assert all_chunks[0].signature_field_name == "answer"
+
+ assert all_chunks[0].chunk == "To"
+ assert all_chunks[1].chunk == " get to the other side of the frying pan!"
+
+ # Start processing the second listened field.
+ assert all_chunks[2].predict_name == "predict2"
+ assert all_chunks[2].signature_field_name == "judgement"
+ assert all_chunks[2].chunk == "The"
+ assert all_chunks[3].chunk == " answer"
+ assert all_chunks[4].chunk == " is"
+ assert all_chunks[5].chunk == " humorous"
+ assert all_chunks[6].chunk == " and"
+ assert all_chunks[7].chunk == " plays on the very funny and classic joke format."
+
+
+async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_stream():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.predict1 = dspy.Predict("question->answer")
+ self.predict2 = dspy.Predict("question,answer->judgement")
+
+ def forward(self, question, **kwargs):
+ answer = self.predict1(question=question, **kwargs).answer
+ judgement = self.predict2(question=question, answer=answer, **kwargs)
+ return judgement
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter())
+
+ async def gemini_stream_1(*args, **kwargs):
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content=" answer ## ]]"))])
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content="To get to the other side."))]
+ )
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))]
+ )
+
+ async def gemini_stream_2(*args, **kwargs):
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ## judgement ## ]]\n\n"))]
+ )
+ yield ModelResponseStream(
+ model="gemini",
+ choices=[
+ StreamingChoices(
+ delta=Delta(
+ content=(
+ "The answer provides the standard punchline for this classic joke format, adapted to the "
+ "specific location mentioned in the question. It is the expected and appropriate response."
+ )
+ )
+ )
+ ],
+ )
+ yield ModelResponseStream(
+ model="gemini",
+ choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))],
+ )
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])
+
+ with mock.patch("litellm.acompletion", new_callable=AsyncMock, side_effect=[gemini_stream_1(), gemini_stream_2()]):
+ program = dspy.streamify(
+ MyProgram(),
+ stream_listeners=[
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="judgement"),
+ ],
+ )
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+
+ assert all_chunks[0].predict_name == "predict1"
+ assert all_chunks[0].signature_field_name == "answer"
+ assert all_chunks[0].chunk == "To get to the other side."
+
+ assert all_chunks[1].predict_name == "predict2"
+ assert all_chunks[1].signature_field_name == "judgement"
+ assert all_chunks[1].chunk == (
+ "The answer provides the standard punchline for this classic joke format, adapted to the specific location "
+ "mentioned in the question. It is the expected and appropriate response."
+ )
+
+
+async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_stream():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.predict1 = dspy.Predict("question->answer")
+ self.predict2 = dspy.Predict("question,answer->judgement")
+
+ def forward(self, question, **kwargs):
+ answer = self.predict1(question=question, **kwargs).answer
+ judgement = self.predict2(question=question, answer=answer, **kwargs)
+ return judgement
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
+
+ async def gemini_stream_1(*args, **kwargs):
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' "answer": "To get to'))]
+ )
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' the other side... of the cutting board!"'))]
+ )
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])
+
+ async def gemini_stream_2(*args, **kwargs):
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])
+ yield ModelResponseStream(
+ model="gemini", choices=[StreamingChoices(delta=Delta(content=' "judgement": "The'))]
+ )
+ yield ModelResponseStream(
+ model="gemini",
+ choices=[
+ StreamingChoices(
+ delta=Delta(
+ content=' answer provides a humorous and relevant punchline to the classic joke setup."'
+ )
+ )
+ ],
+ )
+ yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="}\n"))])
+
+ with mock.patch("litellm.acompletion", new_callable=AsyncMock, side_effect=[gemini_stream_1(), gemini_stream_2()]):
+ program = dspy.streamify(
+ MyProgram(),
+ stream_listeners=[
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="judgement"),
+ ],
+ )
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+
+ assert all_chunks[0].predict_name == "predict1"
+ assert all_chunks[0].signature_field_name == "answer"
+ assert all_chunks[0].chunk == "To get to the other side... of the cutting board!"
+
+ assert all_chunks[1].predict_name == "predict2"
+ assert all_chunks[1].signature_field_name == "judgement"
+ assert all_chunks[1].chunk == "The answer provides a humorous and relevant punchline to the classic joke setup."