Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Explicitly mark the end of streaming (#8587)

Chen Qianab7ac0b91649
diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.py
index 9a73f19b8..4170a0a49 100644
--- a/dspy/streaming/messages.py
+++ b/dspy/streaming/messages.py
@@ -14,6 +14,7 @@ class StreamResponse:
predict_name: str
signature_field_name: str
chunk: str
+ is_last_chunk: bool
@dataclass
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
index 98fd4aca7..851a79193 100644
--- a/dspy/streaming/streaming_listener.py
+++ b/dspy/streaming/streaming_listener.py
@@ -166,7 +166,12 @@ class StreamListener:
token = token.rstrip() # Remove the trailing \n\n
if token:
- return StreamResponse(self.predict_name, self.signature_field_name, token)
+ return StreamResponse(
+ self.predict_name,
+ self.signature_field_name,
+ token,
+ is_last_chunk=self.stream_end,
+ )
def flush(self) -> str:
"""Flush all tokens in the field end queue.
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index fe67bd6da..85ae94573 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -225,9 +225,11 @@ async def test_stream_listener_json_adapter(lm_for_test):
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
+ assert all_chunks[0].is_last_chunk is False
assert all_chunks[-1].predict_name == "predict2"
assert all_chunks[-1].signature_field_name == "judgement"
+ assert all_chunks[-1].is_last_chunk is True
@pytest.mark.anyio
@@ -292,9 +294,11 @@ def test_sync_streaming(lm_for_test):
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
+ assert all_chunks[0].is_last_chunk is False
assert all_chunks[-1].predict_name == "predict2"
assert all_chunks[-1].signature_field_name == "judgement"
+ assert all_chunks[-1].is_last_chunk is True
def test_sync_status_streaming():
@@ -411,6 +415,7 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():
assert all_chunks[3].chunk == " the"
assert all_chunks[4].chunk == " other"
assert all_chunks[5].chunk == " side of the dinner plate!"
+ assert all_chunks[5].is_last_chunk is True
# Start processing the second listened field.
assert all_chunks[6].predict_name == "predict2"
@@ -421,6 +426,8 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():
assert all_chunks[9].chunk == " humorous"
assert all_chunks[10].chunk == " and"
assert all_chunks[11].chunk == " plays"
+ assert all_chunks[11].is_last_chunk is False
+ assert all_chunks[-1].is_last_chunk is True
@pytest.mark.anyio
@@ -763,6 +770,7 @@ async def test_stream_listener_allow_reuse():
# The listener functions twice.
assert concat_message == "To get to the other side!To get to the other side!"
+
@pytest.mark.anyio
async def test_stream_listener_returns_correct_chunk_xml_adapter():
class MyProgram(dspy.Module):