Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Allow custom chunk type in stream (#8679)

Chen Qian2ba68a13330a
diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.py
index 5376d286e..0a1906c94 100644
--- a/dspy/streaming/streamify.py
+++ b/dspy/streaming/streamify.py
@@ -203,6 +203,11 @@ def streamify(
):
yield value
return
+ else:
+ # This wildcard case allows for customized streaming behavior.
+ # It is useful when a users have a custom LM which returns stream chunks in a custom format.
+ # We let those chunks pass through to the user to handle them as needed.
+ yield value
if async_streaming:
return async_streamer
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 85ae94573..2fbd7abd6 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -1,9 +1,11 @@
import asyncio
import time
+from dataclasses import dataclass
from unittest import mock
from unittest.mock import AsyncMock
import pytest
+from asyncer import syncify
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
import dspy
@@ -845,3 +847,30 @@ async def test_stream_listener_returns_correct_chunk_xml_adapter():
assert all_chunks[1].predict_name == "predict2"
assert all_chunks[1].signature_field_name == "judgement"
assert all_chunks[1].chunk == "The answer is humorous."
+
+
+async def test_streaming_allows_custom_chunk_types():
+ @dataclass
+ class CustomChunk:
+ text: str
+
+ class MyProgram(dspy.Module):
+ def forward(self, question, **kwargs):
+ async def send_to_stream():
+ chunk = CustomChunk(text="hello")
+ await dspy.settings.send_stream.send(chunk)
+
+ syncified_send_to_stream = syncify(send_to_stream)
+ syncified_send_to_stream()
+ return dspy.Prediction(answer="dummy output")
+
+ program = dspy.streamify(MyProgram())
+
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ all_chunks.append(value)
+
+ assert isinstance(all_chunks[0], CustomChunk)
+ assert isinstance(all_chunks[1], dspy.Prediction)