Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Support status streaming (#7630)

Chen Qiana0fb6aabfe22
diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.py
index b2f2d053c..b6e953377 100644
--- a/dspy/utils/__init__.py
+++ b/dspy/utils/__init__.py
@@ -1,5 +1,6 @@
from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm
+from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streamify
import os
import requests
@@ -24,4 +25,7 @@ __all__ = [
"DummyLM",
"DummyVectorizer",
"dummy_rm",
+ "StatusMessage",
+ "StatusMessageProvider",
+ "streamify",
]
diff --git a/dspy/utils/asyncify.py b/dspy/utils/asyncify.py
index 0680bc391..7596a03f1 100644
--- a/dspy/utils/asyncify.py
+++ b/dspy/utils/asyncify.py
@@ -1,9 +1,10 @@
-from typing import Any, Awaitable, Callable
+from typing import TYPE_CHECKING, Any, Awaitable, Callable
import asyncer
from anyio import CapacityLimiter
-from dspy.primitives.program import Module
+if TYPE_CHECKING:
+ from dspy.primitives.program import Module
_limiter = None
@@ -26,7 +27,7 @@ def get_limiter():
return _limiter
-def asyncify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:
+def asyncify(program: "Module") -> Callable[[Any, Any], Awaitable[Any]]:
"""
Wraps a DSPy program so that it can be called asynchronously. This is useful for running a
program in parallel with another task (e.g., another DSPy program).
diff --git a/dspy/utils/streaming.py b/dspy/utils/streaming.py
index f51ac11bb..d885591ce 100644
--- a/dspy/utils/streaming.py
+++ b/dspy/utils/streaming.py
@@ -1,23 +1,178 @@
+import logging
from asyncio import iscoroutinefunction
-from typing import Any, AsyncGenerator, Awaitable, Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Dict, Optional
import litellm
import ujson
from anyio import create_memory_object_stream, create_task_group
from anyio.streams.memory import MemoryObjectSendStream
+from asyncer import syncify
+from dspy.dsp.utils.settings import settings
from dspy.primitives.prediction import Prediction
-from dspy.primitives.program import Module
from dspy.utils.asyncify import asyncify
+from dspy.utils.callback import BaseCallback
+logger = logging.getLogger(__name__)
-def streamify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:
+if TYPE_CHECKING:
+ from dspy.primitives.program import Module
+
+
+@dataclass
+class StatusMessage:
+ message: str
+
+
+class StatusMessageProvider:
+ def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ return f"Calling tool {instance.name}..."
+
+ def tool_end_status_message(self, outputs: Any):
+ return "Tool calling finished! Querying the LLM with tool calling results..."
+
+ def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ pass
+
+ def module_end_status_message(self, outputs: Any):
+ pass
+
+ def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ pass
+
+ def lm_end_status_message(self, outputs: Any):
+ pass
+
+
+class StatusStreamingCallback(BaseCallback):
+ def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):
+ self.status_message_provider = status_message_provider or StatusMessageProvider()
+
+ def on_tool_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None or instance.name == "finish":
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.tool_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_tool_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None or outputs == "Completed.":
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.tool_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_lm_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.lm_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_lm_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.lm_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_module_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.module_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_module_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.module_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+
+def streamify(
+ program: "Module",
+ status_message_provider: Optional[StatusMessageProvider] = None,
+) -> Callable[[Any, Any], Awaitable[Any]]:
"""
Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them
- all at once.
+ all at once. It also provides status messages to the user to indicate the progress of the program, and users
+ can implement their own status message provider to customize the status messages and what module to generate
+ status messages for.
Args:
program: The DSPy program to wrap with streaming functionality.
+ status_message_provider: A custom status message generator to use instead of the default one. Users can
+ implement their own status message generator to customize the status messages and what module to generate
+ status messages for.
Returns:
A function that takes the same arguments as the original program, but returns an async
@@ -26,27 +181,68 @@ def streamify(program: Module) -> Callable[[Any, Any], Awaitable[Any]]:
Example:
```python
- class TestSignature(dspy.Signature):
- input_text: str = dspy.InputField()
- output_text: str = dspy.OutputField()
+ import asyncio
+ import dspy
# Create the program and wrap it with streaming functionality
- program = dspy.streamify(dspy.Predict(TestSignature))
+ program = dspy.streamify(dspy.Predict("q->a"))
# Use the program with streaming output
async def use_streaming():
- output_stream = program(input_text="Test")
- async for value in output_stream:
- print(value) # Print each streamed value incrementally
+ output = program(q="Why did a chicken cross the kitchen?")
+ return_value = None
+ async for value in output:
+ if isinstance(value, dspy.Prediction):
+ return_value = value
+ else:
+ print(value)
+ return return_value
+
+ output = asyncio.run(use_streaming())
+ print(output)
```
- """
+
+ Example with custom status message provider:
+ ```python
+ import asyncio
import dspy
+ class MyStatusMessageProvider(StatusMessageProvider):
+ def module_start_status_message(self, instance, inputs):
+ return f"Predicting..."
+
+ def tool_end_status_message(self, outputs):
+ return f"Tool calling finished with output: {outputs}!"
+
+ # Create the program and wrap it with streaming functionality
+ program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())
+
+ # Use the program with streaming output
+ async def use_streaming():
+ output = program(q="Why did a chicken cross the kitchen?")
+ return_value = None
+ async for value in output:
+ if isinstance(value, dspy.Prediction):
+ return_value = value
+ else:
+ print(value)
+ return return_value
+
+ output = asyncio.run(use_streaming())
+ print(output)
+ ```
+ """
if not iscoroutinefunction(program):
program = asyncify(program)
+ callbacks = settings.callbacks
+ status_streaming_callback = StatusStreamingCallback(status_message_provider)
+ if not any(isinstance(c, StatusStreamingCallback) for c in callbacks):
+ callbacks.append(status_streaming_callback)
+ settings.configure(callbacks=callbacks)
+
async def generator(args, kwargs, stream: MemoryObjectSendStream):
- with dspy.settings.context(send_stream=stream):
+ with settings.context(send_stream=stream):
prediction = await program(*args, **kwargs)
await stream.send(prediction)
diff --git a/tests/utils/test_streaming.py b/tests/utils/test_streaming.py
index 2b9c9fd51..3e8907804 100644
--- a/tests/utils/test_streaming.py
+++ b/tests/utils/test_streaming.py
@@ -1,7 +1,7 @@
import pytest
import dspy
-from dspy.utils.streaming import streaming_response
+from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streaming_response
from ..test_utils.server import litellm_test_server
@@ -58,3 +58,67 @@ async def test_streaming_response_yields_expected_response_chunks(litellm_test_s
assert all(chunk.startswith("data: ") for chunk in output_chunks)
assert 'data: {"prediction":{"output_text":"Hello!"}}\n\n' in output_chunks
assert output_chunks[-1] == "data: [DONE]\n\n"
+
+
+async def test_default_status_streaming():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.generate_question = dspy.Tool(lambda x: f"What color is the {x}?", name="generate_question")
+ self.predict = dspy.Predict("question->answer")
+
+ def __call__(self, x: str):
+ question = self.generate_question(x=x)
+ return self.predict(question=question)
+
+ lm = dspy.utils.DummyLM([{"answer": "red"}, {"answer": "blue"}])
+ with dspy.context(lm=lm):
+ program = dspy.streamify(MyProgram())
+ output = program("sky")
+
+ status_messages = []
+ async for value in output:
+ if isinstance(value, StatusMessage):
+ status_messages.append(value)
+
+ assert len(status_messages) == 2
+ assert status_messages[0].message == "Calling tool generate_question..."
+ assert status_messages[1].message == "Tool calling finished! Querying the LLM with tool calling results..."
+
+
+async def test_custom_status_streaming():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.generate_question = dspy.Tool(lambda x: f"What color is the {x}?", name="generate_question")
+ self.predict = dspy.Predict("question->answer")
+
+ def __call__(self, x: str):
+ question = self.generate_question(x=x)
+ return self.predict(question=question)
+
+ class MyStatusMessageProvider(StatusMessageProvider):
+ def tool_start_status_message(self, instance, inputs):
+ return f"Tool starting!"
+
+ def tool_end_status_message(self, outputs):
+ return "Tool finished!"
+
+ def module_start_status_message(self, instance, inputs):
+ if isinstance(instance, dspy.Predict):
+ return "Predict starting!"
+
+ lm = dspy.utils.DummyLM([{"answer": "red"}, {"answer": "blue"}])
+ with dspy.context(lm=lm):
+ program = dspy.streamify(MyProgram(), status_message_provider=MyStatusMessageProvider())
+ output = program("sky")
+
+ status_messages = []
+ async for value in output:
+ if isinstance(value, StatusMessage):
+ status_messages.append(value)
+
+ assert len(status_messages) == 3
+ assert status_messages[0].message == "Tool starting!"
+ assert status_messages[1].message == "Tool finished!"
+ assert status_messages[2].message == "Predict starting!"