Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Generic streaming support (#7856)

Chen Qian6bf0948de5d0
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 3e440344a..a8f800702 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -12,7 +12,7 @@ from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, TwoStepAdapter, Ima
from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging
from dspy.utils.asyncify import asyncify
from dspy.utils.saving import load
-from dspy.utils.streaming import streamify
+from dspy.streaming.streamify import streamify
from dspy.utils.usage_tracker import track_usage
from dspy.dsp.utils.settings import settings
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 0993fb5ca..3f833f8b9 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -1,14 +1,14 @@
import json
import regex
import logging
+from typing import Any, Dict, Type, get_origin
+
+import json_repair
import litellm
import pydantic
-import json_repair
-
-from typing import Any, Dict, Type, get_origin
from pydantic.fields import FieldInfo
-from dspy.clients.lm import LM
+from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName
from dspy.adapters.utils import (
format_field_value,
get_annotation_name,
@@ -16,8 +16,9 @@ from dspy.adapters.utils import (
serialize_for_json,
translate_field_type,
)
+from dspy.clients.lm import LM
+from dspy.dsp.utils.settings import settings
from dspy.signatures.signature import Signature, SignatureMeta
-from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName
logger = logging.getLogger(__name__)
@@ -47,7 +48,11 @@ class JSONAdapter(ChatAdapter):
provider = lm.model.split("/", 1)[0] or "openai"
params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)
- # If response_format is not supported, use basic call.
+ stream_listeners = settings.stream_listeners or []
+ if len(stream_listeners) > 0:
+ raise ValueError("Stream listener is not yet supported for JsonAdapter, please use ChatAdapter instead.")
+
+ # If response_format is not supported, use basic call
if not params or "response_format" not in params:
return super().__call__(lm, lm_kwargs, signature, demos, inputs)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index f878ea640..e70aa8652 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -297,7 +297,10 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac
)
stream = dspy.settings.send_stream
+ caller_predict = dspy.settings.caller_predict
if stream is None:
+ # If `streamify` is not used, or if the exact predict doesn't need to be streamed,
+ # we can just return the completion without streaming.
return litellm.completion(
cache=cache,
**retry_kwargs,
@@ -306,6 +309,7 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac
# The stream is already opened, and will be closed by the caller.
stream = cast(MemoryObjectSendStream, stream)
+ caller_predict_id = id(caller_predict) if caller_predict else None
@syncify
async def stream_completion():
@@ -317,6 +321,9 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac
)
chunks = []
async for chunk in response:
+ if caller_predict_id:
+ # Add the predict id to the chunk so that the stream listener can identify which predict produces it.
+ chunk.predict_id = caller_predict_id
chunks.append(chunk)
await stream.send(chunk)
return litellm.stream_chunk_builder(chunks)
diff --git a/dspy/dsp/utils/settings.py b/dspy/dsp/utils/settings.py
index d4a7f2b9b..cc3262b6f 100644
--- a/dspy/dsp/utils/settings.py
+++ b/dspy/dsp/utils/settings.py
@@ -22,6 +22,8 @@ DEFAULT_CONFIG = dotdict(
disable_history=False,
track_usage=False,
usage_tracker=None,
+ caller_predict=None,
+ stream_listeners=[],
provide_traceback=False, # Whether to include traceback information in error logs.
)
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 8813697ae..618d086cc 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -6,14 +6,13 @@ from pydantic import BaseModel
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.clients.base_lm import BaseLM
from dspy.clients.lm import LM
-from dspy.dsp.utils import settings
+from dspy.dsp.utils.settings import settings
from dspy.predict.parameter import Parameter
from dspy.primitives.prediction import Prediction
from dspy.primitives.program import Module
from dspy.signatures.signature import ensure_signature
from dspy.utils.callback import with_callbacks
-
logger = logging.getLogger(__name__)
@@ -104,13 +103,18 @@ class Predict(Module, Parameter):
)
adapter = settings.adapter or ChatAdapter()
- completions = adapter(
- lm,
- lm_kwargs=config,
- signature=signature,
- demos=demos,
- inputs=kwargs,
- )
+
+ stream_listeners = settings.stream_listeners or []
+ stream = settings.send_stream is not None
+ if stream and len(stream_listeners) > 0:
+ stream = any(stream_listener.predict == self for stream_listener in stream_listeners)
+
+ if stream:
+ with settings.context(caller_predict=self):
+ completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)
+ else:
+ with settings.context(send_stream=None):
+ completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)
pred = Prediction.from_completions(completions, signature=signature)
diff --git a/dspy/streaming/__init__.py b/dspy/streaming/__init__.py
new file mode 100644
index 000000000..7a129d3e3
--- /dev/null
+++ b/dspy/streaming/__init__.py
@@ -0,0 +1,12 @@
+from dspy.streaming.messages import StatusMessage, StatusMessageProvider, StreamResponse
+from dspy.streaming.streamify import streamify, streaming_response
+from dspy.streaming.streaming_listener import StreamListener
+
+__all__ = [
+ "StatusMessage",
+ "StatusMessageProvider",
+ "streamify",
+ "StreamListener",
+ "StreamResponse",
+ "streaming_response",
+]
diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.py
new file mode 100644
index 000000000..bda50341b
--- /dev/null
+++ b/dspy/streaming/messages.py
@@ -0,0 +1,179 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from asyncer import syncify
+
+from dspy.dsp.utils.settings import settings
+from dspy.utils.callback import BaseCallback
+
+
+@dataclass
+class StreamResponse:
+ predict_name: str
+ signature_field_name: str
+ chunk: str
+
+
+@dataclass
+class StatusMessage:
+ """Dataclass that wraps a status message for status streaming."""
+
+ message: str
+
+
+class StatusMessageProvider:
+ """Provides customizable status message streaming for DSPy programs.
+
+ This class serves as a base for creating custom status message providers. Users can subclass
+ and override its methods to define specific status messages for different stages of program execution,
+ each method must return a string.
+
+ Example:
+ ```python
+ class MyStatusMessageProvider(StatusMessageProvider):
+ def lm_start_status_message(self, instance, inputs):
+ return f"Calling LM with inputs {inputs}..."
+
+ def module_end_status_message(self, outputs):
+ return f"Module finished with output: {outputs}!"
+
+ program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())
+ ```
+ """
+
+ def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ """Status message before a `dspy.Tool` is called."""
+ return f"Calling tool {instance.name}..."
+
+ def tool_end_status_message(self, outputs: Any):
+ """Status message after a `dspy.Tool` is called."""
+ return "Tool calling finished! Querying the LLM with tool calling results..."
+
+ def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ """Status message before a `dspy.Module` or `dspy.Predict` is called."""
+ pass
+
+ def module_end_status_message(self, outputs: Any):
+ """Status message after a `dspy.Module` or `dspy.Predict` is called."""
+ pass
+
+ def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
+ """Status message before a `dspy.LM` is called."""
+ pass
+
+ def lm_end_status_message(self, outputs: Any):
+ """Status message after a `dspy.LM` is called."""
+ pass
+
+
+class StatusStreamingCallback(BaseCallback):
+ def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):
+ self.status_message_provider = status_message_provider or StatusMessageProvider()
+
+ def on_tool_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None or instance.name == "finish":
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.tool_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_tool_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None or outputs == "Completed.":
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.tool_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_lm_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.lm_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_lm_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.lm_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_module_start(
+ self,
+ call_id: str,
+ instance: Any,
+ inputs: Dict[str, Any],
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.module_start_status_message(instance, inputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
+
+ def on_module_end(
+ self,
+ call_id: str,
+ outputs: Optional[Dict[str, Any]],
+ exception: Optional[Exception] = None,
+ ):
+ stream = settings.send_stream
+ if stream is None:
+ return
+
+ @syncify
+ async def send_status():
+ status_message = self.status_message_provider.module_end_status_message(outputs)
+ if status_message:
+ await stream.send(StatusMessage(status_message))
+
+ send_status()
diff --git a/dspy/utils/streaming.py b/dspy/streaming/streamify.py
similarity index 50%
rename from dspy/utils/streaming.py
rename to dspy/streaming/streamify.py
index 79f42f9ad..4d4f8bedf 100644
--- a/dspy/utils/streaming.py
+++ b/dspy/streaming/streamify.py
@@ -1,18 +1,18 @@
import logging
from asyncio import iscoroutinefunction
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, List, Optional
import litellm
import ujson
from anyio import create_memory_object_stream, create_task_group
from anyio.streams.memory import MemoryObjectSendStream
-from asyncer import syncify
+from litellm import ModelResponseStream
from dspy.dsp.utils.settings import settings
from dspy.primitives.prediction import Prediction
+from dspy.streaming.messages import StatusMessage, StatusMessageProvider, StatusStreamingCallback
+from dspy.streaming.streaming_listener import StreamListener, find_predictor_for_stream_listeners
from dspy.utils.asyncify import asyncify
-from dspy.utils.callback import BaseCallback
logger = logging.getLogger(__name__)
@@ -20,174 +20,11 @@ if TYPE_CHECKING:
from dspy.primitives.program import Module
-@dataclass
-class StatusMessage:
- """Dataclass that wraps a status message for status streaming."""
-
- message: str
-
-
-class StatusMessageProvider:
- """Provides customizable status message streaming for DSPy programs.
-
- This class serves as a base for creating custom status message providers. Users can subclass
- and override its methods to define specific status messages for different stages of program execution,
- each method must return a string.
-
- Example:
- ```python
- class MyStatusMessageProvider(StatusMessageProvider):
- def lm_start_status_message(self, instance, inputs):
- return f"Calling LM with inputs {inputs}..."
-
- def module_end_status_message(self, outputs):
- return f"Module finished with output: {outputs}!"
-
- program = dspy.streamify(dspy.Predict("q->a"), status_message_provider=MyStatusMessageProvider())
- ```
- """
-
- def tool_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
- """Status message before a `dspy.Tool` is called."""
- return f"Calling tool {instance.name}..."
-
- def tool_end_status_message(self, outputs: Any):
- """Status message after a `dspy.Tool` is called."""
- return "Tool calling finished! Querying the LLM with tool calling results..."
-
- def module_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
- """Status message before a `dspy.Module` or `dspy.Predict` is called."""
- pass
-
- def module_end_status_message(self, outputs: Any):
- """Status message after a `dspy.Module` or `dspy.Predict` is called."""
- pass
-
- def lm_start_status_message(self, instance: Any, inputs: Dict[str, Any]):
- """Status message before a `dspy.LM` is called."""
- pass
-
- def lm_end_status_message(self, outputs: Any):
- """Status message after a `dspy.LM` is called."""
- pass
-
-
-class StatusStreamingCallback(BaseCallback):
- def __init__(self, status_message_provider: Optional[StatusMessageProvider] = None):
- self.status_message_provider = status_message_provider or StatusMessageProvider()
-
- def on_tool_start(
- self,
- call_id: str,
- instance: Any,
- inputs: Dict[str, Any],
- ):
- stream = settings.send_stream
- if stream is None or instance.name == "finish":
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.tool_start_status_message(instance, inputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
- def on_tool_end(
- self,
- call_id: str,
- outputs: Optional[Dict[str, Any]],
- exception: Optional[Exception] = None,
- ):
- stream = settings.send_stream
- if stream is None or outputs == "Completed.":
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.tool_end_status_message(outputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
- def on_lm_start(
- self,
- call_id: str,
- instance: Any,
- inputs: Dict[str, Any],
- ):
- stream = settings.send_stream
- if stream is None:
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.lm_start_status_message(instance, inputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
- def on_lm_end(
- self,
- call_id: str,
- outputs: Optional[Dict[str, Any]],
- exception: Optional[Exception] = None,
- ):
- stream = settings.send_stream
- if stream is None:
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.lm_end_status_message(outputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
- def on_module_start(
- self,
- call_id: str,
- instance: Any,
- inputs: Dict[str, Any],
- ):
- stream = settings.send_stream
- if stream is None:
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.module_start_status_message(instance, inputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
- def on_module_end(
- self,
- call_id: str,
- outputs: Optional[Dict[str, Any]],
- exception: Optional[Exception] = None,
- ):
- stream = settings.send_stream
- if stream is None:
- return
-
- @syncify
- async def send_status():
- status_message = self.status_message_provider.module_end_status_message(outputs)
- if status_message:
- await stream.send(StatusMessage(status_message))
-
- send_status()
-
-
def streamify(
program: "Module",
status_message_provider: Optional[StatusMessageProvider] = None,
+ stream_listeners: Optional[List[StreamListener]] = None,
+ include_final_prediction_in_output_stream: bool = True,
) -> Callable[[Any, Any], Awaitable[Any]]:
"""
Wrap a DSPy program so that it streams its outputs incrementally, rather than returning them
@@ -200,6 +37,12 @@ def streamify(
status_message_provider: A custom status message generator to use instead of the default one. Users can
implement their own status message generator to customize the status messages and what module to generate
status messages for.
+ stream_listeners: A list of stream listeners to capture the streaming output of specific fields of sub predicts
+ in the program. When provided, only the target fields in the target predict will be streamed to the user.
+ include_final_prediction_in_output_stream: Whether to include the final prediction in the output stream, only
+ useful when `stream_listeners` is provided. If `False`, the final prediction will not be included in the
+ output stream. When the program hit cache, or no listeners captured anything, the final prediction will
+ still be included in the output stream even if this is `False`.
Returns:
A function that takes the same arguments as the original program, but returns an async
@@ -211,6 +54,7 @@ def streamify(
import asyncio
import dspy
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini"))
# Create the program and wrap it with streaming functionality
program = dspy.streamify(dspy.Predict("q->a"))
@@ -234,6 +78,8 @@ def streamify(
import asyncio
import dspy
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini"))
+
class MyStatusMessageProvider(StatusMessageProvider):
def module_start_status_message(self, instance, inputs):
return f"Predicting..."
@@ -258,7 +104,45 @@ def streamify(
output = asyncio.run(use_streaming())
print(output)
```
+
+ Example with stream listeners:
+
+ ```python
+ import asyncio
+ import dspy
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
+
+ # Create the program and wrap it with streaming functionality
+ predict = dspy.Predict("question->answer, reasoning")
+ stream_listeners = [
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="reasoning"),
+ ]
+ stream_predict = dspy.streamify(predict, stream_listeners=stream_listeners)
+
+ async def use_streaming():
+ output = stream_predict(
+ question="why did a chicken cross the kitchen?",
+ include_final_prediction_in_output_stream=False,
+ )
+ return_value = None
+ async for value in output:
+ print(value)
+ return return_value
+
+ output = asyncio.run(use_streaming())
+ print(output)
+ ```
+
+ You should see the streaming chunks (in the format of `dspy.streaming.StreamResponse`) in the console output.
"""
+ stream_listeners = stream_listeners or []
+ if len(stream_listeners) > 0:
+ predict_id_to_listener = find_predictor_for_stream_listeners(program, stream_listeners)
+ else:
+ predict_id_to_listener = {}
+
if not iscoroutinefunction(program):
program = asyncify(program)
@@ -268,7 +152,7 @@ def streamify(
callbacks.append(status_streaming_callback)
async def generator(args, kwargs, stream: MemoryObjectSendStream):
- with settings.context(send_stream=stream, callbacks=callbacks):
+ with settings.context(send_stream=stream, callbacks=callbacks, stream_listeners=stream_listeners):
prediction = await program(*args, **kwargs)
await stream.send(prediction)
@@ -279,8 +163,30 @@ def streamify(
tg.start_soon(generator, args, kwargs, send_stream)
async for value in receive_stream:
- yield value
- if isinstance(value, Prediction):
+ if isinstance(value, ModelResponseStream):
+ if len(predict_id_to_listener) == 0:
+ # No listeners are configured, yield the chunk directly for backwards compatibility.
+ yield value
+ else:
+ # We are receiving a chunk from the LM's response stream, delgate it to the listeners to
+ # determine if we should yield a value to the user.
+ output = None
+ for listener in predict_id_to_listener[value.predict_id]:
+ # There should be at most one listener provides a return value.
+ output = listener.receive(value) or output
+ if output:
+ yield output
+ elif isinstance(value, StatusMessage):
+ yield value
+ elif isinstance(value, Prediction):
+ if include_final_prediction_in_output_stream:
+ yield value
+ elif (
+ len(stream_listeners) == 0
+ or any(listener.cache_hit for listener in stream_listeners)
+ or not any(listener.stream_start for listener in stream_listeners)
+ ):
+ yield value
return
return streamer
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
new file mode 100644
index 000000000..7dfc4fa40
--- /dev/null
+++ b/dspy/streaming/streaming_listener.py
@@ -0,0 +1,160 @@
+import re
+from collections import defaultdict
+from queue import Queue
+from typing import TYPE_CHECKING, Any, List
+
+from litellm import ModelResponseStream
+
+from dspy.streaming.messages import StreamResponse
+
+if TYPE_CHECKING:
+ from dspy.primitives.program import Module
+
+
+class StreamListener:
+ """Class that listens to the stream to capture the streeaming of a specific output field of a predictor."""
+
+ def __init__(self, signature_field_name: str, predict: Any = None, predict_name: str = None):
+ """
+ Args:
+ signature_field_name: The name of the field to listen to.
+ predict: The predictor to listen to. If None, when calling `streamify()` it will automatically look for
+ the predictor that has the `signature_field_name` in its signature.
+ predict_name: The name of the predictor to listen to. If None, when calling `streamify()` it will
+ automatically look for the predictor that has the `signature_field_name` in its signature.
+ """
+ self.signature_field_name = signature_field_name
+ self.predict = predict
+ self.predict_name = predict_name
+
+ self.field_start_queue = []
+ self.field_end_queue = Queue()
+ self.stream_start = False
+ self.stream_end = False
+ self.cache_hit = False
+
+ self.start_identifier = f"[[ ## {self.signature_field_name} ## ]]"
+ self.end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")
+
+ def receive(self, chunk: ModelResponseStream):
+ if self.stream_end:
+ return
+
+ try:
+ chunk_message = chunk.choices[0].delta.content
+ if chunk_message is None:
+ return
+ except Exception:
+ return
+
+ if chunk_message and self.start_identifier in chunk_message:
+ # If the cache is hit, the chunk_message could be the full response. When it happens we can
+ # directly end the stream listening.
+ self.cache_hit = True
+ self.stream_start = True
+ self.stream_end = True
+ return
+
+ if len(self.field_start_queue) == 0 and "[[" in chunk_message:
+ # We look for the pattern "[[ ## {self.signature_field_name} ## ]]" to identify the start
+ # of the stream of our target field. Once "[[" is found, we start checking the next tokens
+ # to see if they match our expected identifier.
+ self.field_start_queue.append(chunk_message)
+ return
+
+ if len(self.field_start_queue) > 0 and not self.stream_start:
+ # We keep appending the tokens to the queue until we have a full identifier or the concanated
+ # tokens no longer match our expected identifier.
+ self.field_start_queue.append(chunk_message)
+ concat_message = "".join(self.field_start_queue).strip()
+ start_token_index = concat_message.find("[[")
+ concat_message = concat_message[start_token_index:]
+ if self.start_identifier == concat_message:
+ # We have a full identifier, we can start the stream.
+ self.stream_start = True
+ elif self.start_identifier.startswith(concat_message):
+ # The concanated tokens still match our expected identifier, we keep listening.
+ return
+ else:
+ # Doesn't match the expected identifier, reset the queue.
+ self.field_start_queue = []
+ elif self.stream_start:
+ # The stream is started, we keep returning the token until we see the start of the next field.
+ token = None
+ self.field_end_queue.put(chunk_message)
+ if self.field_end_queue.qsize() > 10:
+ # We keep the last 10 tokens in the buffer to check if they form a valid identifier for
+ # "[[ ## {next_field_name} ## ]]" to identify the end of the current field. In most cases
+ # 10 tokens are enough to cover "[[ ## {next_field_name} ## ]]".
+ token = self.field_end_queue.get()
+ concat_message = "".join(self.field_end_queue.queue).strip()
+ if re.search(self.end_identifier, concat_message):
+ # The next field is identified, we can end the stream and flush out all tokens in the buffer.
+ self.stream_end = True
+ last_token = self.flush()
+ token = token + last_token if token else last_token
+ token = token.rstrip() # Remove the trailing \n\n
+
+ if token:
+ return StreamResponse(self.predict_name, self.signature_field_name, token)
+
+ def flush(self) -> str:
+ """Flush all tokens in the field end queue.
+
+ This method is called to flush out the last a few tokens when the stream is ended. These tokens
+ are in the buffer because we don't directly yield the tokens received by the stream listener
+ with the purpose to not yield the tokens "[[ ## ... ## ]]".
+ """
+ last_tokens = "".join(self.field_end_queue.queue)
+ self.field_end_queue = Queue()
+ start_token_index = last_tokens.find("[[")
+ return last_tokens[:start_token_index]
+
+
+def find_predictor_for_stream_listeners(program: "Module", stream_listeners: List[StreamListener]):
+ """Find the predictor for each stream listener.
+
+ This is a utility function to automatically find the predictor for each stream listener. It is used when some
+ listeners don't specify the predictor they want to listen to. If a listener's `signature_field_name` is not
+ unique in the program, this function will raise an error.
+ """
+ predictors = program.named_predictors()
+
+ field_name_to_named_predictor = {}
+ for listener in stream_listeners:
+ if listener.predict:
+ continue
+ field_name_to_named_predictor[listener.signature_field_name] = None
+
+ for name, predictor in predictors:
+ for field_name, field_info in predictor.signature.output_fields.items():
+ if field_name not in field_name_to_named_predictor:
+ continue
+
+ if field_name_to_named_predictor[field_name] is not None:
+ raise ValueError(
+ f"Signature field {field_name} is not unique in the program, cannot automatically determine which "
+ "predictor to use for streaming. Please specify the predictor to listen to."
+ )
+
+ if field_info.annotation != str:
+ raise ValueError(
+ f"Stream listener can only be applied to string output field, but your field {field_name} is of "
+ f"type {field_info.annotation}."
+ )
+
+ field_name_to_named_predictor[field_name] = (name, predictor)
+
+ predict_id_to_listener = defaultdict(list)
+ for listener in stream_listeners:
+ if listener.predict:
+ continue
+ if listener.signature_field_name not in field_name_to_named_predictor:
+ raise ValueError(
+ f"Signature field {listener.signature_field_name} is not a field of any predictor in the program, "
+ "cannot automatically determine which predictor to use for streaming. Please verify your field name or "
+ "specify the predictor to listen to."
+ )
+ listener.predict_name, listener.predict = field_name_to_named_predictor[listener.signature_field_name]
+ predict_id_to_listener[id(listener.predict)].append(listener)
+ return predict_id_to_listener
diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.py
index b6e953377..84178e491 100644
--- a/dspy/utils/__init__.py
+++ b/dspy/utils/__init__.py
@@ -1,6 +1,6 @@
from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm
-from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streamify
+from dspy.streaming.messages import StatusMessageProvider, StatusMessage
import os
import requests
@@ -27,5 +27,4 @@ __all__ = [
"dummy_rm",
"StatusMessage",
"StatusMessageProvider",
- "streamify",
]
diff --git a/tests/utils/test_streaming.py b/tests/streaming/test_streaming.py
similarity index 74%
rename from tests/utils/test_streaming.py
rename to tests/streaming/test_streaming.py
index 3e8907804..fc89b504e 100644
--- a/tests/utils/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -1,8 +1,9 @@
+import os
import pytest
import dspy
-from dspy.utils.streaming import StatusMessage, StatusMessageProvider, streaming_response
-from ..test_utils.server import litellm_test_server
+from dspy.streaming import StatusMessage, StatusMessageProvider, streaming_response
+from tests.test_utils.server import litellm_test_server
@pytest.mark.anyio
@@ -122,3 +123,40 @@ async def test_custom_status_streaming():
assert status_messages[0].message == "Tool starting!"
assert status_messages[1].message == "Tool finished!"
assert status_messages[2].message == "Predict starting!"
+
+
[email protected](not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")
+async def test_stream_listener():
+ class MyProgram(dspy.Module):
+ def __init__(self):
+ self.predict1 = dspy.Predict("question->answer")
+ self.predict2 = dspy.Predict("question, answer->judgement")
+
+ def __call__(self, x: str, **kwargs):
+ answer = self.predict1(question=x, **kwargs)
+ judgement = self.predict2(question=x, answer=answer, **kwargs)
+ return judgement
+
+ # Turn off the cache to ensure the stream is produced.
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
+ my_program = MyProgram()
+ program = dspy.streamify(
+ my_program,
+ stream_listeners=[
+ dspy.streaming.StreamListener(signature_field_name="answer"),
+ dspy.streaming.StreamListener(signature_field_name="judgement"),
+ ],
+ include_final_prediction_in_output_stream=False,
+ )
+ output = program(x="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+
+ assert all_chunks[0].predict_name == "predict1"
+ assert all_chunks[0].signature_field_name == "answer"
+
+ assert all_chunks[-1].predict_name == "predict2"
+ assert all_chunks[-1].signature_field_name == "judgement"