DSPy: The framework for programming—not prompting—language models
Python
fix(dspy): keep boundary whitespace out of ChatAdapter stream chunks (#10553)
Eugenio "Jay" Zuccarellif4a42885379c
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex 32e9f9eb5..7d5650e54 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -56,6 +56,8 @@ class StreamListener:self.allow_reuse = allow_reuseself.json_adapter_state = {"field_accumulated_messages": ""}+ self.value_started = False+ self.held_whitespace = ""self.adapter_identifiers = {"ChatAdapter": {@@ -136,6 +138,8 @@ class StreamListener:self.field_end_queue = Queue()self.json_adapter_state["field_accumulated_messages"] = ""self.stream_start = False+ self.value_started = False+ self.held_whitespace = ""else:return@@ -302,6 +306,20 @@ class StreamListener:token = token + last_token if token else last_tokentoken = token.rstrip() # Remove the trailing \n\n+ # The parsed field value is stripped, so drop leading whitespace and hold back trailing whitespace until more+ # text arrives. Otherwise the newlines around the field headers leak into the chunks depending on how the+ # provider splits the stream.+ if token and not self.value_started:+ token = token.lstrip()+ if token:+ stripped = token.rstrip()+ if stripped:+ self.value_started = True+ token, self.held_whitespace = self.held_whitespace + stripped, token[len(stripped) :]+ else:+ self.held_whitespace += token+ token = ""+if token or self.stream_end:return StreamResponse(self.predict_name,@@ -355,6 +373,7 @@ class StreamListener:if self.field_end_queue.qsize() > 0:token = self.flush()if token:+ token = self.held_whitespace + tokenreturn StreamResponse(self.predict_name,self.signature_field_name,diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex c71080f57..1233ae7df 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -353,6 +353,40 @@ async def test_streaming_handles_space_correctly():assert "".join([chunk.chunk for chunk in all_chunks]) == "How are you doing?"+ "parts",+ [+ ["[[ ## answer ## ]]\n", "How ", "are ", "you ", "doing?\n\n", "[[ ## completed ## ]]"],+ ["[[ ## answer ## ]]", "\nHow ", "are ", "you ", "doing?", "\n\n", "[[ ## completed ## ]]"],+ ["[[ ## answer", " ## ]]", "\nHow ", "are ", "you ", "doing?", "\n\n[[ ## completed ## ]]"],+ ["[[ ## answer ## ]]\n", "How\n", "are you\n", "doing?", "\n\n[[ ## completed ## ]]"],+ ["[[ ## answer ## ]]\n", "How ", "are ", "you ", "doing?\n\n"],+ ],+)+async def test_streaming_strips_boundary_whitespace_regardless_of_chunking(parts):+ program = dspy.streamify(+ dspy.Predict("question->answer"),+ stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer")],+ )++ async def stream(*args, **kwargs):+ for part in parts:+ yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=part))])++ with mock.patch("litellm.acompletion", side_effect=stream):+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", engine="litellm", cache=False), adapter=dspy.ChatAdapter()):+ all_chunks = []+ final = None+ async for value in program(question="What is the capital of France?"):+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)+ elif isinstance(value, dspy.Prediction):+ final = value++ assert "".join([chunk.chunk for chunk in all_chunks]) == final.answer++@pytest.mark.llm_calldef test_sync_streaming(lm_for_test):class MyProgram(dspy.Module):