Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Make it easier to do sync streaming (#8183)

Chen Qian4771108a3584
diff --git a/docs/docs/tutorials/streaming/index.md b/docs/docs/tutorials/streaming/index.md
index 2799501a0..44a4fbb2c 100644
--- a/docs/docs/tutorials/streaming/index.md
+++ b/docs/docs/tutorials/streaming/index.md
@@ -393,3 +393,38 @@ Final output: Prediction(
sum='9'
)
```
+
+## Synchronous Streaming
+
+By default calling a streamified DSPy program produces an async generator. In order to get back
+a sync generator, you can set the flag `async_streaming=False`:
+
+
+```python
+import os
+
+import dspy
+
+os.environ["OPENAI_API_KEY"] = "your_api_key"
+
+dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"))
+
+predict = dspy.Predict("question->answer")
+
+# Enable streaming for the 'answer' field
+stream_predict = dspy.streamify(
+ predict,
+ stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer")],
+ async_streaming=False,
+)
+
+output = stream_predict(question="why did a chicken cross the kitchen?")
+
+program_output = None
+for chunk in output:
+ if isinstance(chunk, dspy.streaming.StreamResponse):
+ print(chunk)
+ elif isinstance(chunk, dspy.Prediction):
+ program_output = chunk
+print(f"Program output: {program_output}")
+```
\ No newline at end of file
diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.py
index 16359c091..1dfce36f0 100644
--- a/dspy/streaming/streamify.py
+++ b/dspy/streaming/streamify.py
@@ -30,6 +30,7 @@ def streamify(
stream_listeners: Optional[List[StreamListener]] = None,
include_final_prediction_in_output_stream: bool = True,
is_async_program: bool = False,
+ async_streaming: bool = True,
) -> Callable[[Any, Any], Awaitable[Any]]:
"""
Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them
@@ -50,6 +51,8 @@ def streamify(
still be included in the output stream even if this is `False`.
is_async_program: Whether the program is async. If `False`, the program will be wrapped with `asyncify`,
otherwise the program will be called with `acall`.
+ async_streaming: Whether to return an async generator or a sync generator. If `False`, the streaming will be
+ converted to a sync generator.
Returns:
A function that takes the same arguments as the original program, but returns an async
@@ -169,7 +172,7 @@ def streamify(
await stream.send(prediction)
- async def streamer(*args, **kwargs):
+ async def async_streamer(*args, **kwargs):
send_stream, receive_stream = create_memory_object_stream(16)
async with create_task_group() as tg, send_stream, receive_stream:
tg.start_soon(generator, args, kwargs, send_stream)
@@ -201,7 +204,15 @@ def streamify(
yield value
return
- return streamer
+ if async_streaming:
+ return async_streamer
+ else:
+
+ def sync_streamer(*args, **kwargs):
+ output = async_streamer(*args, **kwargs)
+ return apply_sync_streaming(output)
+
+ return sync_streamer
def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 457a39edf..1f89fc229 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -249,11 +249,11 @@ def test_sync_streaming():
dspy.streaming.StreamListener(signature_field_name="judgement"),
],
include_final_prediction_in_output_stream=False,
+ async_streaming=False,
)
output = program(x="why did a chicken cross the kitchen?")
- sync_output = dspy.streaming.apply_sync_streaming(output)
all_chunks = []
- for value in sync_output:
+ for value in output:
if isinstance(value, dspy.streaming.StreamResponse):
all_chunks.append(value)