Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix the request header in streaming mode (#9248)

Chen Qianc811792db1e2
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 9df61ae03..53582e1b7 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -306,6 +306,7 @@ def _get_stream_completion_fn(
request: dict[str, Any],
cache_kwargs: dict[str, Any],
sync=True,
+ headers: dict[str, Any] | None = None,
):
stream = dspy.settings.send_stream
caller_predict = dspy.settings.caller_predict
@@ -321,11 +322,10 @@ def _get_stream_completion_fn(
request["stream_options"] = {"include_usage": True}
async def stream_completion(request: dict[str, Any], cache_kwargs: dict[str, Any]):
- headers = request.pop("headers", None)
response = await litellm.acompletion(
cache=cache_kwargs,
stream=True,
- headers=_get_headers(headers),
+ headers=headers,
**request,
)
chunks = []
@@ -354,14 +354,14 @@ def litellm_completion(request: dict[str, Any], num_retries: int, cache: dict[st
cache = cache or {"no-cache": True, "no-store": True}
request = dict(request)
request.pop("rollout_id", None)
- headers = request.pop("headers", None)
- stream_completion = _get_stream_completion_fn(request, cache, sync=True)
+ headers = _add_dspy_identifier_to_headers(request.pop("headers", None))
+ stream_completion = _get_stream_completion_fn(request, cache, sync=True, headers=headers)
if stream_completion is None:
return litellm.completion(
cache=cache,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=headers,
**request,
)
@@ -393,7 +393,7 @@ def litellm_text_completion(request: dict[str, Any], num_retries: int, cache: di
prompt=prompt,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=_add_dspy_identifier_to_headers(headers),
**request,
)
@@ -409,7 +409,7 @@ async def alitellm_completion(request: dict[str, Any], num_retries: int, cache:
cache=cache,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=_add_dspy_identifier_to_headers(headers),
**request,
)
@@ -439,7 +439,7 @@ async def alitellm_text_completion(request: dict[str, Any], num_retries: int, ca
prompt=prompt,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=_add_dspy_identifier_to_headers(headers),
**request,
)
@@ -455,7 +455,7 @@ def litellm_responses_completion(request: dict[str, Any], num_retries: int, cach
cache=cache,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=_add_dspy_identifier_to_headers(headers),
**request,
)
@@ -471,7 +471,7 @@ async def alitellm_responses_completion(request: dict[str, Any], num_retries: in
cache=cache,
num_retries=num_retries,
retry_strategy="exponential_backoff_retry",
- headers=_get_headers(headers),
+ headers=_add_dspy_identifier_to_headers(headers),
**request,
)
@@ -554,7 +554,7 @@ def _convert_content_item_to_responses_format(item: dict[str, Any]) -> dict[str,
return item
-def _get_headers(headers: dict[str, Any] | None = None):
+def _add_dspy_identifier_to_headers(headers: dict[str, Any] | None = None):
headers = headers or {}
return {
"User-Agent": f"DSPy/{dspy.__version__}",
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 1aa0caad1..15e0e02fd 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -981,3 +981,40 @@ def test_responses_api_with_pydantic_model_input():
"type": "json_schema",
"schema": TestModel.model_json_schema(),
}
+
+
+async def test_streaming_passes_headers_correctly():
+ from dspy.clients.lm import _get_stream_completion_fn
+
+ custom_headers = {"Authorization": "Bearer my-custom-token"}
+ request = {
+ "model": "openai/gpt-4o-mini",
+ "messages": [{"role": "user", "content": "test"}],
+ }
+
+ mock_stream = mock.AsyncMock()
+ mock_stream.send = mock.AsyncMock()
+
+ async def empty_async_generator():
+ return
+ yield # Make it a generator
+
+ with mock.patch("dspy.settings") as mock_settings:
+ mock_settings.send_stream = mock_stream
+ mock_settings.caller_predict = None
+ mock_settings.track_usage = False
+
+ with mock.patch("litellm.acompletion") as mock_acompletion:
+ mock_acompletion.return_value = empty_async_generator()
+
+ stream_fn = _get_stream_completion_fn(request, {}, sync=False, headers=custom_headers)
+ assert stream_fn is not None
+
+ with mock.patch("litellm.stream_chunk_builder", return_value={}):
+ await stream_fn()
+
+ # Verify headers were passed to litellm.acompletion
+ mock_acompletion.assert_called_once()
+ call_kwargs = mock_acompletion.call_args.kwargs
+ assert call_kwargs["headers"]["Authorization"] == "Bearer my-custom-token"