DSPy: The framework for programming—not prompting—language models
Python
Fix the request header in streaming mode (#9248)
Chen Qianc811792db1e2
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 9df61ae03..53582e1b7 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -306,6 +306,7 @@ def _get_stream_completion_fn(request: dict[str, Any],cache_kwargs: dict[str, Any],sync=True,+ headers: dict[str, Any] | None = None,):stream = dspy.settings.send_streamcaller_predict = dspy.settings.caller_predict@@ -321,11 +322,10 @@ def _get_stream_completion_fn(request["stream_options"] = {"include_usage": True}async def stream_completion(request: dict[str, Any], cache_kwargs: dict[str, Any]):- headers = request.pop("headers", None)response = await litellm.acompletion(cache=cache_kwargs,stream=True,- headers=_get_headers(headers),+ headers=headers,**request,)chunks = []@@ -354,14 +354,14 @@ def litellm_completion(request: dict[str, Any], num_retries: int, cache: dict[stcache = cache or {"no-cache": True, "no-store": True}request = dict(request)request.pop("rollout_id", None)- headers = request.pop("headers", None)- stream_completion = _get_stream_completion_fn(request, cache, sync=True)+ headers = _add_dspy_identifier_to_headers(request.pop("headers", None))+ stream_completion = _get_stream_completion_fn(request, cache, sync=True, headers=headers)if stream_completion is None:return litellm.completion(cache=cache,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=headers,**request,)@@ -393,7 +393,7 @@ def litellm_text_completion(request: dict[str, Any], num_retries: int, cache: diprompt=prompt,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=_add_dspy_identifier_to_headers(headers),**request,)@@ -409,7 +409,7 @@ async def alitellm_completion(request: dict[str, Any], num_retries: int, cache:cache=cache,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=_add_dspy_identifier_to_headers(headers),**request,)@@ -439,7 +439,7 @@ async def alitellm_text_completion(request: dict[str, Any], num_retries: int, caprompt=prompt,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=_add_dspy_identifier_to_headers(headers),**request,)@@ -455,7 +455,7 @@ def litellm_responses_completion(request: dict[str, Any], num_retries: int, cachcache=cache,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=_add_dspy_identifier_to_headers(headers),**request,)@@ -471,7 +471,7 @@ async def alitellm_responses_completion(request: dict[str, Any], num_retries: incache=cache,num_retries=num_retries,retry_strategy="exponential_backoff_retry",- headers=_get_headers(headers),+ headers=_add_dspy_identifier_to_headers(headers),**request,)@@ -554,7 +554,7 @@ def _convert_content_item_to_responses_format(item: dict[str, Any]) -> dict[str,return item-def _get_headers(headers: dict[str, Any] | None = None):+def _add_dspy_identifier_to_headers(headers: dict[str, Any] | None = None):headers = headers or {}return {"User-Agent": f"DSPy/{dspy.__version__}",diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 1aa0caad1..15e0e02fd 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -981,3 +981,40 @@ def test_responses_api_with_pydantic_model_input():"type": "json_schema","schema": TestModel.model_json_schema(),}+++async def test_streaming_passes_headers_correctly():+ from dspy.clients.lm import _get_stream_completion_fn++ custom_headers = {"Authorization": "Bearer my-custom-token"}+ request = {+ "model": "openai/gpt-4o-mini",+ "messages": [{"role": "user", "content": "test"}],+ }++ mock_stream = mock.AsyncMock()+ mock_stream.send = mock.AsyncMock()++ async def empty_async_generator():+ return+ yield # Make it a generator++ with mock.patch("dspy.settings") as mock_settings:+ mock_settings.send_stream = mock_stream+ mock_settings.caller_predict = None+ mock_settings.track_usage = False++ with mock.patch("litellm.acompletion") as mock_acompletion:+ mock_acompletion.return_value = empty_async_generator()++ stream_fn = _get_stream_completion_fn(request, {}, sync=False, headers=custom_headers)+ assert stream_fn is not None++ with mock.patch("litellm.stream_chunk_builder", return_value={}):+ await stream_fn()++ # Verify headers were passed to litellm.acompletion+ mock_acompletion.assert_called_once()+ call_kwargs = mock_acompletion.call_args.kwargs+ assert call_kwargs["headers"]["Authorization"] == "Bearer my-custom-token"