DSPy: The framework for programming—not prompting—language models
Python
Make it easier to do sync streaming (#8183)
Chen Qian4771108a3584
diff --git a/docs/docs/tutorials/streaming/index.md b/docs/docs/tutorials/streaming/index.mdindex 2799501a0..44a4fbb2c 100644--- a/docs/docs/tutorials/streaming/index.md+++ b/docs/docs/tutorials/streaming/index.md@@ -393,3 +393,38 @@ Final output: Prediction(sum='9')```++## Synchronous Streaming++By default calling a streamified DSPy program produces an async generator. In order to get back+a sync generator, you can set the flag `async_streaming=False`:+++```python+import os++import dspy++os.environ["OPENAI_API_KEY"] = "your_api_key"++dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"))++predict = dspy.Predict("question->answer")++# Enable streaming for the 'answer' field+stream_predict = dspy.streamify(+ predict,+ stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer")],+ async_streaming=False,+)++output = stream_predict(question="why did a chicken cross the kitchen?")++program_output = None+for chunk in output:+ if isinstance(chunk, dspy.streaming.StreamResponse):+ print(chunk)+ elif isinstance(chunk, dspy.Prediction):+ program_output = chunk+print(f"Program output: {program_output}")+```\ No newline at end of filediff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.pyindex 16359c091..1dfce36f0 100644--- a/dspy/streaming/streamify.py+++ b/dspy/streaming/streamify.py@@ -30,6 +30,7 @@ def streamify(stream_listeners: Optional[List[StreamListener]] = None,include_final_prediction_in_output_stream: bool = True,is_async_program: bool = False,+ async_streaming: bool = True,) -> Callable[[Any, Any], Awaitable[Any]]:"""Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them@@ -50,6 +51,8 @@ def streamify(still be included in the output stream even if this is `False`.is_async_program: Whether the program is async. If `False`, the program will be wrapped with `asyncify`,otherwise the program will be called with `acall`.+ async_streaming: Whether to return an async generator or a sync generator. If `False`, the streaming will be+ converted to a sync generator.Returns:A function that takes the same arguments as the original program, but returns an async@@ -169,7 +172,7 @@ def streamify(await stream.send(prediction)- async def streamer(*args, **kwargs):+ async def async_streamer(*args, **kwargs):send_stream, receive_stream = create_memory_object_stream(16)async with create_task_group() as tg, send_stream, receive_stream:tg.start_soon(generator, args, kwargs, send_stream)@@ -201,7 +204,15 @@ def streamify(yield valuereturn- return streamer+ if async_streaming:+ return async_streamer+ else:++ def sync_streamer(*args, **kwargs):+ output = async_streamer(*args, **kwargs)+ return apply_sync_streaming(output)++ return sync_streamerdef apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 457a39edf..1f89fc229 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -249,11 +249,11 @@ def test_sync_streaming():dspy.streaming.StreamListener(signature_field_name="judgement"),],include_final_prediction_in_output_stream=False,+ async_streaming=False,)output = program(x="why did a chicken cross the kitchen?")- sync_output = dspy.streaming.apply_sync_streaming(output)all_chunks = []- for value in sync_output:+ for value in output:if isinstance(value, dspy.streaming.StreamResponse):all_chunks.append(value)