DSPy: The framework for programming—not prompting—language models
Python
Generic streaming support (#7856)
Chen Qian6bf0948de5d0
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 3e440344a..a8f800702 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -12,7 +12,7 @@ from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, TwoStepAdapter, Imafrom dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_loggingfrom dspy.utils.asyncify import asyncifyfrom dspy.utils.saving import load-from dspy.utils.streaming import streamify+from dspy.streaming.streamify import streamifyfrom dspy.utils.usage_tracker import track_usagefrom dspy.dsp.utils.settings import settingsdiff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.pyindex 0993fb5ca..3f833f8b9 100644--- a/dspy/adapters/json_adapter.py+++ b/dspy/adapters/json_adapter.py@@ -1,14 +1,14 @@import jsonimport regeximport logging+from typing import Any, Dict, Type, get_origin++import json_repairimport litellmimport pydantic-import json_repair--from typing import Any, Dict, Type, get_originfrom pydantic.fields import FieldInfo-from dspy.clients.lm import LM+from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithNamefrom dspy.adapters.utils import (format_field_value,get_annotation_name,@@ -16,8 +16,9 @@ from dspy.adapters.utils import (serialize_for_json,translate_field_type,)+from dspy.clients.lm import LM+from dspy.dsp.utils.settings import settingsfrom dspy.signatures.signature import Signature, SignatureMeta-from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithNamelogger = logging.getLogger(__name__)@@ -47,7 +48,11 @@ class JSONAdapter(ChatAdapter):provider = lm.model.split("/", 1)[0] or "openai"params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)- # If response_format is not supported, use basic call.+ stream_listeners = settings.stream_listeners or []+ if len(stream_listeners) > 0:+ raise ValueError("Stream listener is not yet supported for JsonAdapter, please use ChatAdapter instead.")++ # If response_format is not supported, use basic callif not params or "response_format" not in params:return super().__call__(lm, lm_kwargs, signature, demos, inputs)diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex f878ea640..e70aa8652 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -297,7 +297,10 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac)stream = dspy.settings.send_stream+ caller_predict = dspy.settings.caller_predictif stream is None:+ # If `streamify` is not used, or if the exact predict doesn't need to be streamed,+ # we can just return the completion without streaming.return litellm.completion(cache=cache,**retry_kwargs,@@ -306,6 +309,7 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac# The stream is already opened, and will be closed by the caller.stream = cast(MemoryObjectSendStream, stream)+ caller_predict_id = id(caller_predict) if caller_predict else None@syncifyasync def stream_completion():@@ -317,6 +321,9 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac)chunks = []async for chunk in response:+ if caller_predict_id:+ # Add the predict id to the chunk so that the stream listener can identify which predict produces it.+ chunk.predict_id = caller_predict_idchunks.append(chunk)await stream.send(chunk)return litellm.stream_chunk_builder(chunks)diff --git a/dspy/dsp/utils/settings.py b/dspy/dsp/utils/settings.pyindex d4a7f2b9b..cc3262b6f 100644--- a/dspy/dsp/utils/settings.py+++ b/dspy/dsp/utils/settings.py@@ -22,6 +22,8 @@ DEFAULT_CONFIG = dotdict(disable_history=False,track_usage=False,usage_tracker=None,+ caller_predict=None,+ stream_listeners=[],provide_traceback=False, # Whether to include traceback information in error logs.)diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 8813697ae..618d086cc 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -6,14 +6,13 @@ from pydantic import BaseModelfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.clients.base_lm import BaseLMfrom dspy.clients.lm import LM-from dspy.dsp.utils import settings+from dspy.dsp.utils.settings import settingsfrom dspy.predict.parameter import Parameterfrom dspy.primitives.prediction import Predictionfrom dspy.primitives.program import Modulefrom dspy.signatures.signature import ensure_signaturefrom dspy.utils.callback import with_callbacks-logger = logging.getLogger(__name__)@@ -104,13 +103,18 @@ class Predict(Module, Parameter):)adapter = settings.adapter or ChatAdapter()- completions = adapter(- lm,- lm_kwargs=config,- signature=signature,- demos=demos,- inputs=kwargs,- )++ stream_listeners = settings.stream_listeners or []+ stream = settings.send_stream is not None+ if stream and len(stream_listeners) > 0:+ stream = any(stream_listener.predict == self for stream_listener in stream_listeners)++ if stream:+ with settings.context(caller_predict=self):+ completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)+ else:+ with settings.context(send_stream=None):+ completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)pred = Prediction.from_completions(completions, signature=signature)diff --git a/dspy/streaming/__init__.py b/dspy/streaming/__init__.pynew file mode 100644index 000000000..7a129d3e3--- /dev/null+++ b/dspy/streaming/__init__.py@@ -0,0 +1,12 @@+from dspy.streaming.messages import StatusMessage, StatusMessageProvider, StreamResponse+from dspy.streaming.streamify import streamify, streaming_response+from dspy.streaming.streaming_listener import StreamListener++__all__ = [+ "StatusMessage",+ "StatusMessageProvider",+ "streamify",+ "StreamListener",+ "StreamResponse",+ "streaming_response",+]diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.pynew file mode 100644index 000000000..bda50341b--- /dev/null+++ b/dspy/streaming/messages.py@@ -0,0 +1,179 @@+from dataclasses import dataclass+from typing import Any, Dict, Optional++from asyncer import syncify++from dspy.dsp.utils.settings import settings+from dspy.utils.callback import BaseCallback+++@dataclass+class StreamResponse:+ predict_name: str+ signature_field_name: str+ chunk: str+++@dataclass+class StatusMessage:+ """Dataclass that wraps a status message for status streaming."""++ message: str+++class StatusMessageProvider:+ """Provides customizable status message streaming for DSPy programs.++ This class serves as a base for creating custom status message providers. Users can subclass+ and override its methods to define specific status messages for different stages of program execution,+ each method must return a string.++ Example:+ ```python+ class MyStatusMessageProvider(StatusMessageProvider):+ def lm_start_status_message(self, instance, inputs):+ return f"Calling LM with inputs {inputs}..."++ def module_end_status_message(self, outputs):+ return f"Module finished with output: {outputs}!"++ program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())+ ```+ """++ def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ """Status message before a `dspy.Tool` is called."""+ return f"Calling tool {instance.name}..."++ def tool_end_status_message(self, outputs: Any):+ """Status message after a `dspy.Tool` is called."""+ return "Tool calling finished! Querying the LLM with tool calling results..."++ def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ """Status message before a `dspy.Module` or `dspy.Predict` is called."""+ pass++ def module_end_status_message(self, outputs: Any):+ """Status message after a `dspy.Module` or `dspy.Predict` is called."""+ pass++ def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):+ """Status message before a `dspy.LM` is called."""+ pass++ def lm_end_status_message(self, outputs: Any):+ """Status message after a `dspy.LM` is called."""+ pass+++class StatusStreamingCallback(BaseCallback):+ def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):+ self.status_message_provider = status_message_provider or StatusMessageProvider()++ def on_tool_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None or instance.name == "finish":+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.tool_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_tool_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None or outputs == "Completed.":+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.tool_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_lm_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.lm_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_lm_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.lm_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_module_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.module_start_status_message(instance, inputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()++ def on_module_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ stream = settings.send_stream+ if stream is None:+ return++ @syncify+ async def send_status():+ status_message = self.status_message_provider.module_end_status_message(outputs)+ if status_message:+ await stream.send(StatusMessage(status_message))++ send_status()diff --git a/dspy/utils/streaming.py b/dspy/streaming/streamify.pysimilarity index 50%rename from dspy/utils/streaming.pyrename to dspy/streaming/streamify.pyindex 79f42f9ad..4d4f8bedf 100644--- a/dspy/utils/streaming.py+++ b/dspy/streaming/streamify.py@@ -1,18 +1,18 @@import loggingfrom asyncio import iscoroutinefunction-from dataclasses import dataclass-from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Dict, Optional+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, List, Optionalimport litellmimport ujsonfrom anyio import create_memory_object_stream, create_task_groupfrom anyio.streams.memory import MemoryObjectSendStream-from asyncer import syncify+from litellm import ModelResponseStreamfrom dspy.dsp.utils.settings import settingsfrom dspy.primitives.prediction import Prediction+from dspy.streaming.messages import StatusMessage, StatusMessageProvider, StatusStreamingCallback+from dspy.streaming.streaming_listener import StreamListener, find_predictor_for_stream_listenersfrom dspy.utils.asyncify import asyncify-from dspy.utils.callback import BaseCallbacklogger = logging.getLogger(__name__)@@ -20,174 +20,11 @@ if TYPE_CHECKING:from dspy.primitives.program import Module-@dataclass-class StatusMessage:- """Dataclass that wraps a status message for status streaming."""-- message: str---class StatusMessageProvider:- """Provides customizable status message streaming for DSPy programs.-- This class serves as a base for creating custom status message providers. Users can subclass- and override its methods to define specific status messages for different stages of program execution,- each method must return a string.-- Example:- ```python- class MyStatusMessageProvider(StatusMessageProvider):- def lm_start_status_message(self, instance, inputs):- return f"Calling LM with inputs {inputs}..."-- def module_end_status_message(self, outputs):- return f"Module finished with output: {outputs}!"-- program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())- ```- """-- def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):- """Status message before a `dspy.Tool` is called."""- return f"Calling tool {instance.name}..."-- def tool_end_status_message(self, outputs: Any):- """Status message after a `dspy.Tool` is called."""- return "Tool calling finished! Querying the LLM with tool calling results..."-- def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):- """Status message before a `dspy.Module` or `dspy.Predict` is called."""- pass-- def module_end_status_message(self, outputs: Any):- """Status message after a `dspy.Module` or `dspy.Predict` is called."""- pass-- def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):- """Status message before a `dspy.LM` is called."""- pass-- def lm_end_status_message(self, outputs: Any):- """Status message after a `dspy.LM` is called."""- pass---class StatusStreamingCallback(BaseCallback):- def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):- self.status_message_provider = status_message_provider or StatusMessageProvider()-- def on_tool_start(- self,- call_id: str,- instance: Any,- inputs: Dict[str, Any],- ):- stream = settings.send_stream- if stream is None or instance.name == "finish":- return-- @syncify- async def send_status():- status_message = self.status_message_provider.tool_start_status_message(instance, inputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()-- def on_tool_end(- self,- call_id: str,- outputs: Optional[Dict[str, Any]],- exception: Optional[Exception] = None,- ):- stream = settings.send_stream- if stream is None or outputs == "Completed.":- return-- @syncify- async def send_status():- status_message = self.status_message_provider.tool_end_status_message(outputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()-- def on_lm_start(- self,- call_id: str,- instance: Any,- inputs: Dict[str, Any],- ):- stream = settings.send_stream- if stream is None:- return-- @syncify- async def send_status():- status_message = self.status_message_provider.lm_start_status_message(instance, inputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()-- def on_lm_end(- self,- call_id: str,- outputs: Optional[Dict[str, Any]],- exception: Optional[Exception] = None,- ):- stream = settings.send_stream- if stream is None:- return-- @syncify- async def send_status():- status_message = self.status_message_provider.lm_end_status_message(outputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()-- def on_module_start(- self,- call_id: str,- instance: Any,- inputs: Dict[str, Any],- ):- stream = settings.send_stream- if stream is None:- return-- @syncify- async def send_status():- status_message = self.status_message_provider.module_start_status_message(instance, inputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()-- def on_module_end(- self,- call_id: str,- outputs: Optional[Dict[str, Any]],- exception: Optional[Exception] = None,- ):- stream = settings.send_stream- if stream is None:- return-- @syncify- async def send_status():- status_message = self.status_message_provider.module_end_status_message(outputs)- if status_message:- await stream.send(StatusMessage(status_message))-- send_status()--def streamify(program: "Module",status_message_provider: Optional[StatusMessageProvider] = None,+ stream_listeners: Optional[List[StreamListener]] = None,+ include_final_prediction_in_output_stream: bool = True,) -> Callable[[Any, Any], Awaitable[Any]]:"""Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them@@ -200,6 +37,12 @@ def streamify(status_message_provider: A custom status message generator to use instead of the default one. Users canimplement their own status message generator to customize the status messages and what module to generatestatus messages for.+ stream_listeners: A list of stream listeners to capture the streaming output of specific fields of sub predicts+ in the program. When provided, only the target fields in the target predict will be streamed to the user.+ include_final_prediction_in_output_stream: Whether to include the final prediction in the output stream, only+ useful when `stream_listeners` is provided. If `False`, the final prediction will not be included in the+ output stream. When the program hit cache, or no listeners captured anything, the final prediction will+ still be included in the output stream even if this is `False`.Returns:A function that takes the same arguments as the original program, but returns an async@@ -211,6 +54,7 @@ def streamify(import asyncioimport dspy+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini"))# Create the program and wrap it with streaming functionalityprogram = dspy.streamify(dspy.Predict("q->a"))@@ -234,6 +78,8 @@ def streamify(import asyncioimport dspy+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini"))+class MyStatusMessageProvider(StatusMessageProvider):def module_start_status_message(self, instance, inputs):return f"Predicting..."@@ -258,7 +104,45 @@ def streamify(output = asyncio.run(use_streaming())print(output)```++ Example with stream listeners:++ ```python+ import asyncio+ import dspy++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))++ # Create the program and wrap it with streaming functionality+ predict = dspy.Predict("question->answer, reasoning")+ stream_listeners = [+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="reasoning"),+ ]+ stream_predict = dspy.streamify(predict, stream_listeners=stream_listeners)++ async def use_streaming():+ output = stream_predict(+ question="why did a chicken cross the kitchen?",+ include_final_prediction_in_output_stream=False,+ )+ return_value = None+ async for value in output:+ print(value)+ return return_value++ output = asyncio.run(use_streaming())+ print(output)+ ```++ You should see the streaming chunks (in the format of `dspy.streaming.StreamResponse`) in the console output."""+ stream_listeners = stream_listeners or []+ if len(stream_listeners) > 0:+ predict_id_to_listener = find_predictor_for_stream_listeners(program, stream_listeners)+ else:+ predict_id_to_listener = {}+if not iscoroutinefunction(program):program = asyncify(program)@@ -268,7 +152,7 @@ def streamify(callbacks.append(status_streaming_callback)async def generator(args, kwargs, stream: MemoryObjectSendStream):- with settings.context(send_stream=stream, callbacks=callbacks):+ with settings.context(send_stream=stream, callbacks=callbacks, stream_listeners=stream_listeners):prediction = await program(*args, **kwargs)await stream.send(prediction)@@ -279,8 +163,30 @@ def streamify(tg.start_soon(generator, args, kwargs, send_stream)async for value in receive_stream:- yield value- if isinstance(value, Prediction):+ if isinstance(value, ModelResponseStream):+ if len(predict_id_to_listener) == 0:+ # No listeners are configured, yield the chunk directly for backwards compatibility.+ yield value+ else:+ # We are receiving a chunk from the LM's response stream, delgate it to the listeners to+ # determine if we should yield a value to the user.+ output = None+ for listener in predict_id_to_listener[value.predict_id]:+ # There should be at most one listener provides a return value.+ output = listener.receive(value) or output+ if output:+ yield output+ elif isinstance(value, StatusMessage):+ yield value+ elif isinstance(value, Prediction):+ if include_final_prediction_in_output_stream:+ yield value+ elif (+ len(stream_listeners) == 0+ or any(listener.cache_hit for listener in stream_listeners)+ or not any(listener.stream_start for listener in stream_listeners)+ ):+ yield valuereturnreturn streamerdiff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pynew file mode 100644index 000000000..7dfc4fa40--- /dev/null+++ b/dspy/streaming/streaming_listener.py@@ -0,0 +1,160 @@+import re+from collections import defaultdict+from queue import Queue+from typing import TYPE_CHECKING, Any, List++from litellm import ModelResponseStream++from dspy.streaming.messages import StreamResponse++if TYPE_CHECKING:+ from dspy.primitives.program import Module+++class StreamListener:+ """Class that listens to the stream to capture the streeaming of a specific output field of a predictor."""++ def __init__(self, signature_field_name: str, predict: Any = None, predict_name: str = None):+ """+ Args:+ signature_field_name: The name of the field to listen to.+ predict: The predictor to listen to. If None, when calling `streamify()` it will automatically look for+ the predictor that has the `signature_field_name` in its signature.+ predict_name: The name of the predictor to listen to. If None, when calling `streamify()` it will+ automatically look for the predictor that has the `signature_field_name` in its signature.+ """+ self.signature_field_name = signature_field_name+ self.predict = predict+ self.predict_name = predict_name++ self.field_start_queue = []+ self.field_end_queue = Queue()+ self.stream_start = False+ self.stream_end = False+ self.cache_hit = False++ self.start_identifier = f"[[ ## {self.signature_field_name} ## ]]"+ self.end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")++ def receive(self, chunk: ModelResponseStream):+ if self.stream_end:+ return++ try:+ chunk_message = chunk.choices[0].delta.content+ if chunk_message is None:+ return+ except Exception:+ return++ if chunk_message and self.start_identifier in chunk_message:+ # If the cache is hit, the chunk_message could be the full response. When it happens we can+ # directly end the stream listening.+ self.cache_hit = True+ self.stream_start = True+ self.stream_end = True+ return++ if len(self.field_start_queue) == 0 and "[[" in chunk_message:+ # We look for the pattern "[[ ## {self.signature_field_name} ## ]]" to identify the start+ # of the stream of our target field. Once "[[" is found, we start checking the next tokens+ # to see if they match our expected identifier.+ self.field_start_queue.append(chunk_message)+ return++ if len(self.field_start_queue) > 0 and not self.stream_start:+ # We keep appending the tokens to the queue until we have a full identifier or the concanated+ # tokens no longer match our expected identifier.+ self.field_start_queue.append(chunk_message)+ concat_message = "".join(self.field_start_queue).strip()+ start_token_index = concat_message.find("[[")+ concat_message = concat_message[start_token_index:]+ if self.start_identifier == concat_message:+ # We have a full identifier, we can start the stream.+ self.stream_start = True+ elif self.start_identifier.startswith(concat_message):+ # The concanated tokens still match our expected identifier, we keep listening.+ return+ else:+ # Doesn't match the expected identifier, reset the queue.+ self.field_start_queue = []+ elif self.stream_start:+ # The stream is started, we keep returning the token until we see the start of the next field.+ token = None+ self.field_end_queue.put(chunk_message)+ if self.field_end_queue.qsize() > 10:+ # We keep the last 10 tokens in the buffer to check if they form a valid identifier for+ # "[[ ## {next_field_name} ## ]]" to identify the end of the current field. In most cases+ # 10 tokens are enough to cover "[[ ## {next_field_name} ## ]]".+ token = self.field_end_queue.get()+ concat_message = "".join(self.field_end_queue.queue).strip()+ if re.search(self.end_identifier, concat_message):+ # The next field is identified, we can end the stream and flush out all tokens in the buffer.+ self.stream_end = True+ last_token = self.flush()+ token = token + last_token if token else last_token+ token = token.rstrip() # Remove the trailing \n\n++ if token:+ return StreamResponse(self.predict_name, self.signature_field_name, token)++ def flush(self) -> str:+ """Flush all tokens in the field end queue.++ This method is called to flush out the last a few tokens when the stream is ended. These tokens+ are in the buffer because we don't directly yield the tokens received by the stream listener+ with the purpose to not yield the tokens "[[ ## ... ## ]]".+ """+ last_tokens = "".join(self.field_end_queue.queue)+ self.field_end_queue = Queue()+ start_token_index = last_tokens.find("[[")+ return last_tokens[:start_token_index]+++def find_predictor_for_stream_listeners(program: "Module", stream_listeners: List[StreamListener]):+ """Find the predictor for each stream listener.++ This is a utility function to automatically find the predictor for each stream listener. It is used when some+ listeners don't specify the predictor they want to listen to. If a listener's `signature_field_name` is not+ unique in the program, this function will raise an error.+ """+ predictors = program.named_predictors()++ field_name_to_named_predictor = {}+ for listener in stream_listeners:+ if listener.predict:+ continue+ field_name_to_named_predictor[listener.signature_field_name] = None++ for name, predictor in predictors:+ for field_name, field_info in predictor.signature.output_fields.items():+ if field_name not in field_name_to_named_predictor:+ continue++ if field_name_to_named_predictor[field_name] is not None:+ raise ValueError(+ f"Signature field {field_name} is not unique in the program, cannot automatically determine which "+ "predictor to use for streaming. Please specify the predictor to listen to."+ )++ if field_info.annotation != str:+ raise ValueError(+ f"Stream listener can only be applied to string output field, but your field {field_name} is of "+ f"type {field_info.annotation}."+ )++ field_name_to_named_predictor[field_name] = (name, predictor)++ predict_id_to_listener = defaultdict(list)+ for listener in stream_listeners:+ if listener.predict:+ continue+ if listener.signature_field_name not in field_name_to_named_predictor:+ raise ValueError(+ f"Signature field {listener.signature_field_name} is not a field of any predictor in the program, "+ "cannot automatically determine which predictor to use for streaming. Please verify your field name or "+ "specify the predictor to listen to."+ )+ listener.predict_name, listener.predict = field_name_to_named_predictor[listener.signature_field_name]+ predict_id_to_listener[id(listener.predict)].append(listener)+ return predict_id_to_listenerdiff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.pyindex b6e953377..84178e491 100644--- a/dspy/utils/__init__.py+++ b/dspy/utils/__init__.py@@ -1,6 +1,6 @@from dspy.utils.callback import BaseCallback, with_callbacksfrom dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm-from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streamify+from dspy.streaming.messages import StatusMessageProvider, StatusMessageimport osimport requests@@ -27,5 +27,4 @@ __all__ = ["dummy_rm","StatusMessage","StatusMessageProvider",- "streamify",]diff --git a/tests/utils/test_streaming.py b/tests/streaming/test_streaming.pysimilarity index 74%rename from tests/utils/test_streaming.pyrename to tests/streaming/test_streaming.pyindex 3e8907804..fc89b504e 100644--- a/tests/utils/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -1,8 +1,9 @@+import osimport pytestimport dspy-from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streaming_response-from ..test_utils.server import litellm_test_server+from dspy.streaming import StatusMessage, StatusMessageProvider, streaming_response+from tests.test_utils.server import litellm_test_server@pytest.mark.anyio@@ -122,3 +123,40 @@ async def test_custom_status_streaming():assert status_messages[0].message == "Tool starting!"assert status_messages[1].message == "Tool finished!"assert status_messages[2].message == "Predict starting!"++[email protected](not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")+async def test_stream_listener():+ class MyProgram(dspy.Module):+ def __init__(self):+ self.predict1 = dspy.Predict("question->answer")+ self.predict2 = dspy.Predict("question, answer->judgement")++ def __call__(self, x: str, **kwargs):+ answer = self.predict1(question=x, **kwargs)+ judgement = self.predict2(question=x, answer=answer, **kwargs)+ return judgement++ # Turn off the cache to ensure the stream is produced.+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))+ my_program = MyProgram()+ program = dspy.streamify(+ my_program,+ stream_listeners=[+ dspy.streaming.StreamListener(signature_field_name="answer"),+ dspy.streaming.StreamListener(signature_field_name="judgement"),+ ],+ include_final_prediction_in_output_stream=False,+ )+ output = program(x="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)++ assert all_chunks[0].predict_name == "predict1"+ assert all_chunks[0].signature_field_name == "answer"++ assert all_chunks[-1].predict_name == "predict2"+ assert all_chunks[-1].signature_field_name == "judgement"