DSPy: The framework for programming—not prompting—language models
Python
Allow custom chunk type in stream (#8679)
Chen Qian2ba68a13330a
diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.pyindex 5376d286e..0a1906c94 100644--- a/dspy/streaming/streamify.py+++ b/dspy/streaming/streamify.py@@ -203,6 +203,11 @@ def streamify():yield valuereturn+ else:+ # This wildcard case allows for customized streaming behavior.+ # It is useful when a users have a custom LM which returns stream chunks in a custom format.+ # We let those chunks pass through to the user to handle them as needed.+ yield valueif async_streaming:return async_streamerdiff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 85ae94573..2fbd7abd6 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -1,9 +1,11 @@import asyncioimport time+from dataclasses import dataclassfrom unittest import mockfrom unittest.mock import AsyncMockimport pytest+from asyncer import syncifyfrom litellm.types.utils import Delta, ModelResponseStream, StreamingChoicesimport dspy@@ -845,3 +847,30 @@ async def test_stream_listener_returns_correct_chunk_xml_adapter():assert all_chunks[1].predict_name == "predict2"assert all_chunks[1].signature_field_name == "judgement"assert all_chunks[1].chunk == "The answer is humorous."+++async def test_streaming_allows_custom_chunk_types():+ @dataclass+ class CustomChunk:+ text: str++ class MyProgram(dspy.Module):+ def forward(self, question, **kwargs):+ async def send_to_stream():+ chunk = CustomChunk(text="hello")+ await dspy.settings.send_stream.send(chunk)++ syncified_send_to_stream = syncify(send_to_stream)+ syncified_send_to_stream()+ return dspy.Prediction(answer="dummy output")++ program = dspy.streamify(MyProgram())++ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ all_chunks.append(value)++ assert isinstance(all_chunks[0], CustomChunk)+ assert isinstance(all_chunks[1], dspy.Prediction)