Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(dspy): keep boundary whitespace out of ChatAdapter stream chunks (#10553)

Eugenio "Jay" Zuccarellif4a42885379c
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
index 32e9f9eb5..7d5650e54 100644
--- a/dspy/streaming/streaming_listener.py
+++ b/dspy/streaming/streaming_listener.py
@@ -56,6 +56,8 @@ class StreamListener:
self.allow_reuse = allow_reuse
self.json_adapter_state = {"field_accumulated_messages": ""}
+ self.value_started = False
+ self.held_whitespace = ""
self.adapter_identifiers = {
"ChatAdapter": {
@@ -136,6 +138,8 @@ class StreamListener:
self.field_end_queue = Queue()
self.json_adapter_state["field_accumulated_messages"] = ""
self.stream_start = False
+ self.value_started = False
+ self.held_whitespace = ""
else:
return
@@ -302,6 +306,20 @@ class StreamListener:
token = token + last_token if token else last_token
token = token.rstrip() # Remove the trailing \n\n
+ # The parsed field value is stripped, so drop leading whitespace and hold back trailing whitespace until more
+ # text arrives. Otherwise the newlines around the field headers leak into the chunks depending on how the
+ # provider splits the stream.
+ if token and not self.value_started:
+ token = token.lstrip()
+ if token:
+ stripped = token.rstrip()
+ if stripped:
+ self.value_started = True
+ token, self.held_whitespace = self.held_whitespace + stripped, token[len(stripped) :]
+ else:
+ self.held_whitespace += token
+ token = ""
+
if token or self.stream_end:
return StreamResponse(
self.predict_name,
@@ -355,6 +373,7 @@ class StreamListener:
if self.field_end_queue.qsize() > 0:
token = self.flush()
if token:
+ token = self.held_whitespace + token
return StreamResponse(
self.predict_name,
self.signature_field_name,
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index c71080f57..1233ae7df 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -353,6 +353,40 @@ async def test_streaming_handles_space_correctly():
assert "".join([chunk.chunk for chunk in all_chunks]) == "How are you doing?"
+ "parts",
+ [
+ ["[[ ## answer ## ]]\n", "How ", "are ", "you ", "doing?\n\n", "[[ ## completed ## ]]"],
+ ["[[ ## answer ## ]]", "\nHow ", "are ", "you ", "doing?", "\n\n", "[[ ## completed ## ]]"],
+ ["[[ ## answer", " ## ]]", "\nHow ", "are ", "you ", "doing?", "\n\n[[ ## completed ## ]]"],
+ ["[[ ## answer ## ]]\n", "How\n", "are you\n", "doing?", "\n\n[[ ## completed ## ]]"],
+ ["[[ ## answer ## ]]\n", "How ", "are ", "you ", "doing?\n\n"],
+ ],
+)
+async def test_streaming_strips_boundary_whitespace_regardless_of_chunking(parts):
+ program = dspy.streamify(
+ dspy.Predict("question->answer"),
+ stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer")],
+ )
+
+ async def stream(*args, **kwargs):
+ for part in parts:
+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=part))])
+
+ with mock.patch("litellm.acompletion", side_effect=stream):
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", engine="litellm", cache=False), adapter=dspy.ChatAdapter()):
+ all_chunks = []
+ final = None
+ async for value in program(question="What is the capital of France?"):
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
+ elif isinstance(value, dspy.Prediction):
+ final = value
+
+ assert "".join([chunk.chunk for chunk in all_chunks]) == final.answer
+
+
@pytest.mark.llm_call
def test_sync_streaming(lm_for_test):
class MyProgram(dspy.Module):