DSPy: The framework for programming—not prompting—language models
Python
Support status streaming (#7630)
Chen Qiana0fb6aabfe22
diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.pyindex b2f2d053c..b6e953377 100644--- a/dspy/utils/__init__.py+++ b/dspy/utils/__init__.py@@ -1,5 +1,6 @@from dspy.utils.callback import BaseCallback, with_callbacksfrom dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm+from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streamifyimport osimport requests@@ -24,4 +25,7 @@ __all__ = ["DummyLM","DummyVectorizer","dummy_rm",+ "StatusMessage",+ "StatusMessageProvider",+ "streamify",]diff --git a/dspy/utils/asyncify.py b/dspy/utils/asyncify.pyindex 0680bc391..7596a03f1 100644--- a/dspy/utils/asyncify.py+++ b/dspy/utils/asyncify.py@@ -1,9 +1,10 @@-from typing import Any, Awaitable, Callable+from typing import TYPE_CHECKING, Any, Awaitable, Callableimport asyncerfrom anyio import CapacityLimiter-from dspy.primitives.program import Module+if TYPE_CHECKING:+ from dspy.primitives.program import Module_limiter = None@@ -26,7 +27,7 @@ def get_limiter():return _limiter-def asyncify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:+def asyncify(program: "Module") -> Callable[[Any, Any], Awaitable[Any]]:"""Wraps a DSPy program so that it can be called asynchronously. This is useful for running aprogram in parallel with another task (e.g., another DSPy program).diff --git a/dspy/utils/streaming.py b/dspy/utils/streaming.pyindex f51ac11bb..d885591ce 100644--- a/dspy/utils/streaming.py+++ b/dspy/utils/streaming.py@@ -1,23 +1,178 @@+import loggingfrom asyncio import iscoroutinefunction-from typing import Any, AsyncGenerator, Awaitable, Callable+from dataclasses import dataclass+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Dict, Optionalimport litellmimport ujsonfrom anyio import create_memory_object_stream, create_task_groupfrom anyio.streams.memory import MemoryObjectSendStream+from asyncer import syncify+from dspy.dsp.utils.settings import settingsfrom dspy.primitives.prediction import Prediction-from dspy.primitives.program import Modulefrom dspy.utils.asyncify import asyncify+from dspy.utils.callback import BaseCallback+logger = logging.getLogger(__name__)-def streamify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:+if TYPE_CHECKING:+ from dspy.primitives.program import Module+++@dataclass+class StatusMessage:+ message: str+++class StatusMessageProvider:+ def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ return f"Calling tool {instance.name}..."++ def tool_end_status_message(self, outputs: Any):+ return "Tool calling finished! Querying the LLM with tool calling results..."++ def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ pass++ def module_end_status_message(self, outputs: Any):+ pass++ def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ pass++ def lm_end_status_message(self, outputs: Any):+ pass+++class StatusStreamingCallback(BaseCallback):+ def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):+ self.status_message_provider = status_message_provider or StatusMessageProvider()++ def on_tool_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None or instance.name == "finish":+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.tool_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_tool_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None or outputs == "Completed.":+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.tool_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_lm_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.lm_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_lm_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.lm_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_module_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.module_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_module_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.module_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()+++def streamify(+ program: "Module",+ status_message_provider: Optional[StatusMessageProvider] = None,+) -> Callable[[Any, Any], Awaitable[Any]]:"""Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them- all at once.+ all at once. It also provides status messages to the user to indicate the progress of the program, and users+ can implement their own status message provider to customize the status messages and what module to generate+ status messages for.Args:program: The DSPy program to wrap with streaming functionality.+ status_message_provider: A custom status message generator to use instead of the default one. Users can+ implement their own status message generator to customize the status messages and what module to generate+ status messages for.Returns:A function that takes the same arguments as the original program, but returns an async@@ -26,27 +181,68 @@ def streamify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:Example:```python- class TestSignature(dspy.Signature):- input_text: str = dspy.InputField()- output_text: str = dspy.OutputField()+ import asyncio+ import dspy# Create the program and wrap it with streaming functionality- program = dspy.streamify(dspy.Predict(TestSignature))+ program = dspy.streamify(dspy.Predict("q->a"))# Use the program with streaming outputasync def use_streaming():- output_stream = program(input_text="Test")- async for value in output_stream:- print(value) # Print each streamed value incrementally+ output = program(q="Why did a chicken cross the kitchen?")+ return_value = None+ async for value in output:+ if isinstance(value, dspy.Prediction):+ return_value = value+ else:+ print(value)+ return return_value++ output = asyncio.run(use_streaming())+ print(output)```- """++ Example with custom status message provider:+ ```python+ import asyncioimport dspy+ class MyStatusMessageProvider(StatusMessageProvider):+ def module_start_status_message(self, instance, inputs):+ return f"Predicting..."++ def tool_end_status_message(self, outputs):+ return f"Tool calling finished with output: {outputs}!"++ # Create the program and wrap it with streaming functionality+ program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())++ # Use the program with streaming output+ async def use_streaming():+ output = program(q="Why did a chicken cross the kitchen?")+ return_value = None+ async for value in output:+ if isinstance(value, dspy.Prediction):+ return_value = value+ else:+ print(value)+ return return_value++ output = asyncio.run(use_streaming())+ print(output)+ ```+ """if not iscoroutinefunction(program):program = asyncify(program)+ callbacks = settings.callbacks+ status_streaming_callback = StatusStreamingCallback(status_message_provider)+ if not any(isinstance(c, StatusStreamingCallback) for c in callbacks):+ callbacks.append(status_streaming_callback)+ settings.configure(callbacks=callbacks)+async def generator(args, kwargs, stream: MemoryObjectSendStream):- with dspy.settings.context(send_stream=stream):+ with settings.context(send_stream=stream):prediction = await program(*args, **kwargs)await stream.send(prediction)diff --git a/tests/utils/test_streaming.py b/tests/utils/test_streaming.pyindex 2b9c9fd51..3e8907804 100644--- a/tests/utils/test_streaming.py+++ b/tests/utils/test_streaming.py@@ -1,7 +1,7 @@import pytestimport dspy-from dspy.utils.streaming import streaming_response+from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streaming_responsefrom ..test_utils.server import litellm_test_server@@ -58,3 +58,67 @@ async def test_streaming_response_yields_expected_response_chunks(litellm_test_sassert all(chunk.startswith("data: ") for chunk in output_chunks)assert 'data: {"prediction":{"output_text":"Hello!"}}\n\n' in output_chunksassert output_chunks[-1] == "data: [DONE]\n\n"+++async def test_default_status_streaming():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.generate_question = dspy.Tool(lambda x: f"What color is the {x}?", name="generate_question")+ self.predict = dspy.Predict("question->answer")++ def __call__(self, x: str):+ question = self.generate_question(x=x)+ return self.predict(question=question)++ lm = dspy.utils.DummyLM([{"answer": "red"}, {"answer": "blue"}])+ with dspy.context(lm=lm):+ program = dspy.streamify(MyProgram())+ output = program("sky")++ status_messages = []+ async for value in output:+ if isinstance(value, StatusMessage):+ status_messages.append(value)++ assert len(status_messages) == 2+ assert status_messages[0].message == "Calling tool generate_question..."+ assert status_messages[1].message == "Tool calling finished! Querying the LLM with tool calling results..."+++async def test_custom_status_streaming():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.generate_question = dspy.Tool(lambda x: f"What color is the {x}?", name="generate_question")+ self.predict = dspy.Predict("question->answer")++ def __call__(self, x: str):+ question = self.generate_question(x=x)+ return self.predict(question=question)++ class MyStatusMessageProvider(StatusMessageProvider):+ def tool_start_status_message(self, instance, inputs):+ return f"Tool starting!"++ def tool_end_status_message(self, outputs):+ return "Tool finished!"++ def module_start_status_message(self, instance, inputs):+ if isinstance(instance, dspy.Predict):+ return "Predict starting!"++ lm = dspy.utils.DummyLM([{"answer": "red"}, {"answer": "blue"}])+ with dspy.context(lm=lm):+ program = dspy.streamify(MyProgram(), status_message_provider=MyStatusMessageProvider())+ output = program("sky")++ status_messages = []+ async for value in output:+ if isinstance(value, StatusMessage):+ status_messages.append(value)++ assert len(status_messages) == 3+ assert status_messages[0].message == "Tool starting!"+ assert status_messages[1].message == "Tool finished!"+ assert status_messages[2].message == "Predict starting!"