DSPy: The framework for programming—not prompting—language models
Python
Utility that converts async stream to sync stream (#8162)
Chen Qiand89028f7e2b8
diff --git a/dspy/streaming/__init__.py b/dspy/streaming/__init__.pyindex 7a129d3e3..9d5ae8057 100644--- a/dspy/streaming/__init__.py+++ b/dspy/streaming/__init__.py@@ -1,5 +1,5 @@from dspy.streaming.messages import StatusMessage, StatusMessageProvider, StreamResponse-from dspy.streaming.streamify import streamify, streaming_response+from dspy.streaming.streamify import apply_sync_streaming, streamify, streaming_responsefrom dspy.streaming.streaming_listener import StreamListener__all__ = [@@ -9,4 +9,5 @@ __all__ = ["StreamListener","StreamResponse","streaming_response",+ "apply_sync_streaming",]diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.pyindex c99f2a154..16359c091 100644--- a/dspy/streaming/streamify.py+++ b/dspy/streaming/streamify.py@@ -1,6 +1,10 @@+import asyncio+import contextvarsimport logging+import threadingfrom asyncio import iscoroutinefunction-from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, List, Optional+from queue import Queue+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Generator, List, Optionalimport litellmimport ujson@@ -200,6 +204,46 @@ def streamify(return streamer+def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:+ """Convert the async streaming generator to a sync generator."""+ queue = Queue() # Queue to hold items from the async generator+ stop_sentinel = object() # Sentinel to signal the generator is complete++ # To propagate prediction request ID context to the child thread+ context = contextvars.copy_context()+ from dspy.dsp.utils.settings import thread_local_overrides++ parent_overrides = thread_local_overrides.overrides.copy()++ def producer():+ """Runs in a background thread to fetch items asynchronously."""++ original_overrides = thread_local_overrides.overrides+ thread_local_overrides.overrides = parent_overrides.copy()++ async def runner():+ try:+ async for item in async_generator:+ queue.put(item)+ finally:+ # Signal completion+ queue.put(stop_sentinel)++ context.run(asyncio.run, runner())+ thread_local_overrides.overrides = original_overrides++ # Start the producer in a background thread+ thread = threading.Thread(target=producer, daemon=True)+ thread.start()++ # Consume items from the queue+ while True:+ item = queue.get() # Block until an item is available+ if item is stop_sentinel:+ break+ yield item++async def streaming_response(streamer: AsyncGenerator) -> AsyncGenerator:"""Convert a DSPy program output stream to an OpenAI-compatible output stream that can bediff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 9daf559d2..457a39edf 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -225,3 +225,65 @@ async def test_stream_listener_json_adapter():assert all_chunks[-1].predict_name == "predict2"assert all_chunks[-1].signature_field_name == "judgement"++[email protected](not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")+def test_sync_streaming():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question, answer->judgement")++ def __call__(self, x: str, **kwargs):+ answer = self.predict1(question=x, **kwargs)+ judgement = self.predict2(question=x, answer=answer, **kwargs)+ return judgement++ # Turn off the cache to ensure the stream is produced.+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))+ my_program = MyProgram()+ program = dspy.streamify(+ my_program,+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ include_final_prediction_in_output_stream=False,+ )+ output = program(x="why did a chicken cross the kitchen?")+ sync_output = dspy.streaming.apply_sync_streaming(output)+ all_chunks = []+ for value in sync_output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"++ assert all_chunks[-1].predict_name == "predict2"+ assert all_chunks[-1].signature_field_name == "judgement"+++def test_sync_status_streaming():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.generate_question = dspy.Tool(lambda x: f"What color is the {x}?", name="generate_question")+ self.predict = dspy.Predict("question->answer")++ def __call__(self, x: str):+ question = self.generate_question(x=x)+ return self.predict(question=question)++ lm = dspy.utils.DummyLM([{"answer": "red"}, {"answer": "blue"}])+ with dspy.context(lm=lm):+ program = dspy.streamify(MyProgram())+ output = program("sky")+ sync_output = dspy.streaming.apply_sync_streaming(output)+ status_messages = []+ for value in sync_output:+ if isinstance(value, StatusMessage):+ status_messages.append(value)++ assert len(status_messages) == 2+ assert status_messages[0].message == "Calling tool generate_question..."+ assert status_messages[1].message == "Tool calling finished! Querying the LLM with tool calling results..."