Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Merge async settings change into main (#8361)

Chen Qiandd8cf1cb284d
diff --git a/dspy/dsp/utils/settings.py b/dspy/dsp/utils/settings.py
index fc0f7300d..cf1c7d267 100644
--- a/dspy/dsp/utils/settings.py
+++ b/dspy/dsp/utils/settings.py
@@ -1,3 +1,5 @@
+import asyncio
+import contextvars
import copy
import threading
from contextlib import contextmanager
@@ -38,13 +40,7 @@ config_owner_thread_id = None
# Global lock for settings configuration
global_lock = threading.Lock()
-
-class ThreadLocalOverrides(threading.local):
- def __init__(self):
- self.overrides = dotdict()
-
-
-thread_local_overrides = ThreadLocalOverrides()
+thread_local_overrides = contextvars.ContextVar("context_overrides", default=dotdict())
class Settings:
@@ -75,7 +71,7 @@ class Settings:
return global_lock
def __getattr__(self, name):
- overrides = getattr(thread_local_overrides, "overrides", dotdict())
+ overrides = thread_local_overrides.get()
if name in overrides:
return overrides[name]
elif name in main_thread_config:
@@ -96,7 +92,7 @@ class Settings:
self.__setattr__(key, value)
def __contains__(self, key):
- overrides = getattr(thread_local_overrides, "overrides", dotdict())
+ overrides = thread_local_overrides.get()
return key in overrides or key in main_thread_config
def get(self, key, default=None):
@@ -106,23 +102,60 @@ class Settings:
return default
def copy(self):
- overrides = getattr(thread_local_overrides, "overrides", dotdict())
+ overrides = thread_local_overrides.get()
return dotdict({**main_thread_config, **overrides})
@property
def config(self):
return self.copy()
- def configure(self, **kwargs):
+ def _ensure_configure_allowed(self):
global main_thread_config, config_owner_thread_id
current_thread_id = threading.get_ident()
- with self.lock:
- # First configuration: establish ownership. If ownership established, only that thread can configure.
- if config_owner_thread_id in [None, current_thread_id]:
- config_owner_thread_id = current_thread_id
- else:
- raise RuntimeError("dspy.settings can only be changed by the thread that initially configured it.")
+ if config_owner_thread_id is None:
+ # First `configure` call is always allowed.
+ config_owner_thread_id = current_thread_id
+ return
+
+ if config_owner_thread_id != current_thread_id:
+ # Disallow a second `configure` calls from other threads.
+ raise RuntimeError("dspy.settings can only be changed by the thread that initially configured it.")
+
+ # Async task doesn't allow a second `configure` call, must use dspy.context(...) instead.
+ is_async_task = False
+ try:
+ if asyncio.current_task() is not None:
+ is_async_task = True
+ except RuntimeError:
+ # This exception (e.g., "no current task") means we are not in an async loop/task,
+ # or asyncio module itself is not fully functional in this specific sub-thread context.
+ is_async_task = False
+
+ if not is_async_task:
+ return
+
+ # We are in an async task. Now check for IPython and allow calling `configure` from IPython.
+ in_ipython = False
+ try:
+ from IPython import get_ipython
+
+ # get_ipython is a global injected by IPython environments.
+ # We check its existence and type to be more robust.
+ in_ipython = get_ipython() is not None
+ except Exception:
+ # If `IPython` is not installed or `get_ipython` failed, we are not in an IPython environment.
+ in_ipython = False
+
+ if not in_ipython:
+ raise RuntimeError(
+ "dspy.settings.configure(...) cannot be called a second time from an async task. Use "
+ "`dspy.context(...)` instead."
+ )
+
+ def configure(self, **kwargs):
+ # If no exception is raised, the `configure` call is allowed.
+ self._ensure_configure_allowed()
# Update global config
for k, v in kwargs.items():
@@ -136,17 +169,17 @@ class Settings:
If threads are spawned inside this block using ParallelExecutor, they will inherit these overrides.
"""
- original_overrides = getattr(thread_local_overrides, "overrides", dotdict()).copy()
+ original_overrides = thread_local_overrides.get().copy()
new_overrides = dotdict({**main_thread_config, **original_overrides, **kwargs})
- thread_local_overrides.overrides = new_overrides
+ token = thread_local_overrides.set(new_overrides)
try:
yield
finally:
- thread_local_overrides.overrides = original_overrides
+ thread_local_overrides.reset(token)
def __repr__(self):
- overrides = getattr(thread_local_overrides, "overrides", dotdict())
+ overrides = thread_local_overrides.get()
combined_config = {**main_thread_config, **overrides}
return repr(combined_config)
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index f39cc8912..e306591e8 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -2,7 +2,7 @@ from typing import Optional
import magicattr
-from dspy.dsp.utils.settings import settings
+from dspy.dsp.utils.settings import settings, thread_local_overrides
from dspy.predict.parallel import Parallel
from dspy.primitives.module import BaseModule
from dspy.utils.callback import with_callbacks
@@ -51,7 +51,7 @@ class Module(BaseModule, metaclass=ProgramMeta):
caller_modules.append(self)
with settings.context(caller_modules=caller_modules):
- if settings.track_usage and settings.usage_tracker is None:
+ if settings.track_usage and thread_local_overrides.get().get("usage_tracker") is None:
with track_usage() as usage_tracker:
output = self.forward(*args, **kwargs)
output.set_lm_usage(usage_tracker.get_total_tokens())
@@ -66,7 +66,7 @@ class Module(BaseModule, metaclass=ProgramMeta):
caller_modules.append(self)
with settings.context(caller_modules=caller_modules):
- if settings.track_usage and settings.usage_tracker is None:
+ if settings.track_usage and thread_local_overrides.get().get("usage_tracker") is None:
with track_usage() as usage_tracker:
output = await self.aforward(*args, **kwargs)
output.set_lm_usage(usage_tracker.get_total_tokens())
diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.py
index 98adeacf4..534f60fcc 100644
--- a/dspy/streaming/streamify.py
+++ b/dspy/streaming/streamify.py
@@ -222,16 +222,10 @@ def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:
# To propagate prediction request ID context to the child thread
context = contextvars.copy_context()
- from dspy.dsp.utils.settings import thread_local_overrides
-
- parent_overrides = thread_local_overrides.overrides.copy()
def producer():
"""Runs in a background thread to fetch items asynchronously."""
- original_overrides = thread_local_overrides.overrides
- thread_local_overrides.overrides = parent_overrides.copy()
-
async def runner():
try:
async for item in async_generator:
@@ -241,7 +235,6 @@ def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:
queue.put(stop_sentinel)
context.run(asyncio.run, runner())
- thread_local_overrides.overrides = original_overrides
# Start the producer in a background thread
thread = threading.Thread(target=producer, daemon=True)
diff --git a/dspy/utils/asyncify.py b/dspy/utils/asyncify.py
index 7596a03f1..5ca9cdf32 100644
--- a/dspy/utils/asyncify.py
+++ b/dspy/utils/asyncify.py
@@ -46,17 +46,17 @@ def asyncify(program: "Module") -> Callable[[Any, Any], Awaitable[Any]]:
# Capture the current overrides at call-time.
from dspy.dsp.utils.settings import thread_local_overrides
- parent_overrides = thread_local_overrides.overrides.copy()
+ parent_overrides = thread_local_overrides.get().copy()
def wrapped_program(*a, **kw):
from dspy.dsp.utils.settings import thread_local_overrides
- original_overrides = thread_local_overrides.overrides
- thread_local_overrides.overrides = parent_overrides.copy()
+ original_overrides = thread_local_overrides.get()
+ token = thread_local_overrides.set({**original_overrides, **parent_overrides.copy()})
try:
return program(*a, **kw)
finally:
- thread_local_overrides.overrides = original_overrides
+ thread_local_overrides.reset(token)
# Create a fresh asyncified callable each time, ensuring the latest context is used.
call_async = asyncer.asyncify(wrapped_program, abandon_on_cancel=True, limiter=get_limiter())
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.py
index 9ad76493f..d45df534e 100644
--- a/dspy/utils/parallelizer.py
+++ b/dspy/utils/parallelizer.py
@@ -86,8 +86,8 @@ class ParallelExecutor:
# Apply parent's thread-local overrides
from dspy.dsp.utils.settings import thread_local_overrides
- original = thread_local_overrides.overrides
- thread_local_overrides.overrides = parent_overrides.copy()
+ original = thread_local_overrides.get()
+ token = thread_local_overrides.set({**original, **parent_overrides.copy()})
if parent_overrides.get("usage_tracker"):
# Usage tracker needs to be deep copied across threads so that each thread tracks its own usage
thread_local_overrides.overrides["usage_tracker"] = copy.deepcopy(parent_overrides["usage_tracker"])
@@ -95,7 +95,7 @@ class ParallelExecutor:
try:
return index, function(item)
finally:
- thread_local_overrides.overrides = original
+ thread_local_overrides.reset(token)
# Handle Ctrl-C in the main thread
@contextlib.contextmanager
@@ -121,7 +121,7 @@ class ParallelExecutor:
with interrupt_manager():
from dspy.dsp.utils.settings import thread_local_overrides
- parent_overrides = thread_local_overrides.overrides.copy()
+ parent_overrides = thread_local_overrides.get().copy()
futures_map = {}
futures_set = set()
diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py
index c7a9460a3..f045188fb 100644
--- a/tests/adapters/test_two_step_adapter.py
+++ b/tests/adapters/test_two_step_adapter.py
@@ -94,9 +94,8 @@ async def test_two_step_adapter_async_call():
mock_extraction_lm.kwargs = {"temperature": 1.0}
mock_extraction_lm.model = "openai/gpt-4o"
- dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))
-
- result = await program.acall(question="What is 5 + 7?")
+ with dspy.context(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm)):
+ result = await program.acall(question="What is 5 + 7?")
assert result.answer == 12
diff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.py
index b3df2b0cd..365266ca9 100644
--- a/tests/callback/test_callback.py
+++ b/tests/callback/test_callback.py
@@ -189,13 +189,12 @@ def test_callback_complex_module():
@pytest.mark.asyncio
async def test_callback_async_module():
callback = MyCallback()
- dspy.settings.configure(
+ with dspy.context(
lm=DummyLM({"How are you?": {"answer": "test output", "reasoning": "No more responses"}}),
callbacks=[callback],
- )
-
- cot = dspy.ChainOfThought("question -> answer", n=3)
- result = await cot.acall(question="How are you?")
+ ):
+ cot = dspy.ChainOfThought("question -> answer", n=3)
+ result = await cot.acall(question="How are you?")
assert result["answer"] == "test output"
assert result["reasoning"] == "No more responses"
diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.py
index 68f8e4c9b..247256dba 100644
--- a/tests/predict/test_chain_of_thought.py
+++ b/tests/predict/test_chain_of_thought.py
@@ -19,7 +19,7 @@ def test_initialization_with_string_signature():
@pytest.mark.asyncio
async def test_async_chain_of_thought():
lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])
- dspy.settings.configure(lm=lm)
- program = ChainOfThought("question -> answer")
- result = await program.acall(question="What is 1+1?")
- assert result.answer == "2"
+ with dspy.context(lm=lm):
+ program = ChainOfThought("question -> answer")
+ result = await program.acall(question="What is 1+1?")
+ assert result.answer == "2"
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index d4ce96158..c1aea9727 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -1,5 +1,8 @@
+import asyncio
import copy
import enum
+import time
+import types
from datetime import datetime
from unittest.mock import patch
@@ -506,6 +509,69 @@ def test_lm_usage():
assert result.get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+def test_lm_usage_with_parallel():
+ program = Predict("question -> answer")
+
+ def program_wrapper(question):
+ # Sleep to make it possible to cause a race condition
+ time.sleep(0.5)
+ return program(question=question)
+
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True)
+ with patch(
+ "dspy.clients.lm.litellm_completion",
+ return_value=ModelResponse(
+ choices=[{"message": {"content": "[[ ## answer ## ]]\nParis"}}],
+ usage={"total_tokens": 10},
+ ),
+ ):
+ parallelizer = dspy.Parallel()
+ input_pairs = [
+ (program_wrapper, {"question": "What is the capital of France?"}),
+ (program_wrapper, {"question": "What is the capital of France?"}),
+ ]
+ results = parallelizer(input_pairs)
+ assert results[0].answer == "Paris"
+ assert results[1].answer == "Paris"
+ assert results[0].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+ assert results[1].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+
+
+async def test_lm_usage_with_async():
+ program = Predict("question -> answer")
+
+ original_aforward = program.aforward
+
+ async def patched_aforward(self, **kwargs):
+ await asyncio.sleep(1)
+ return await original_aforward(**kwargs)
+
+ program.aforward = types.MethodType(patched_aforward, program)
+
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True):
+ with patch(
+ "litellm.acompletion",
+ return_value=ModelResponse(
+ choices=[{"message": {"content": "[[ ## answer ## ]]\nParis"}}],
+ usage={"total_tokens": 10},
+ ),
+ ):
+ coroutines = [
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ ]
+ results = await asyncio.gather(*coroutines)
+ assert results[0].answer == "Paris"
+ assert results[1].answer == "Paris"
+ assert results[0].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+ assert results[1].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+ assert results[2].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+ assert results[3].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10
+
+
def test_positional_arguments():
program = Predict("question -> answer")
with pytest.raises(ValueError) as e:
@@ -569,9 +635,9 @@ def test_field_constraints(adapter_type):
@pytest.mark.asyncio
async def test_async_predict():
program = Predict("question -> answer")
- dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]))
- result = await program.acall(question="What is the capital of France?")
- assert result.answer == "Paris"
+ with dspy.context(lm=DummyLM([{"answer": "Paris"}])):
+ result = await program.acall(question="What is the capital of France?")
+ assert result.answer == "Paris"
def test_predicted_outputs_piped_from_predict_to_lm_call():
diff --git a/tests/predict/test_react.py b/tests/predict/test_react.py
index e3beb300d..4ae1b6f52 100644
--- a/tests/predict/test_react.py
+++ b/tests/predict/test_react.py
@@ -254,16 +254,15 @@ async def test_async_tool_calling_with_pydantic_args():
},
]
)
- dspy.settings.configure(lm=lm)
-
- outputs = await react.acall(
- participant_name="Alice",
- event_info=CalendarEvent(
- name="Science Fair",
- date="Friday",
- participants={"Alice": "female", "Bob": "male"},
- ),
- )
+ with dspy.context(lm=lm):
+ outputs = await react.acall(
+ participant_name="Alice",
+ event_info=CalendarEvent(
+ name="Science Fair",
+ date="Friday",
+ participants={"Alice": "female", "Bob": "male"},
+ ),
+ )
assert outputs.invitation_letter == "It's my honor to invite Alice to the Science Fair event on Friday."
expected_trajectory = {
@@ -309,9 +308,8 @@ async def test_async_error_retry():
{"reasoning": "I added the numbers successfully", "c": 3},
]
)
- dspy.settings.configure(lm=lm)
-
- outputs = await react.acall(a=1, b=2, max_iters=2)
+ with dspy.context(lm=lm):
+ outputs = await react.acall(a=1, b=2, max_iters=2)
traj = outputs.trajectory
# Exact-match checks (thoughts + tool calls)
diff --git a/tests/primitives/test_module.py b/tests/primitives/test_module.py
index bc1d2975c..544146839 100644
--- a/tests/primitives/test_module.py
+++ b/tests/primitives/test_module.py
@@ -1,3 +1,4 @@
+import asyncio
import logging
import os
import threading
@@ -5,6 +6,7 @@ from unittest.mock import patch
import pytest
from litellm import Choices, Message, ModelResponse
+from litellm.types.utils import Usage
import dspy
from dspy.utils.dummies import DummyLM
@@ -307,6 +309,54 @@ def test_usage_tracker_in_parallel():
assert results[1].get_lm_usage().keys() == set(["openai/gpt-3.5-turbo"])
+async def test_usage_tracker_async_parallel():
+ program = dspy.Predict("question -> answer")
+
+ with patch("litellm.acompletion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],
+ usage=Usage(
+ **{
+ "prompt_tokens": 1117,
+ "completion_tokens": 46,
+ "total_tokens": 1163,
+ "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
+ "completion_tokens_details": {
+ "reasoning_tokens": 0,
+ "audio_tokens": 0,
+ "accepted_prediction_tokens": 0,
+ "rejected_prediction_tokens": 0,
+ },
+ },
+ ),
+ model="openai/gpt-4o-mini",
+ )
+
+ coroutines = [
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ program.acall(question="What is the capital of France?"),
+ ]
+ with dspy.settings.context(
+ lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True, adapter=dspy.JSONAdapter()
+ ):
+ results = await asyncio.gather(*coroutines)
+
+ assert results[0].get_lm_usage() is not None
+ assert results[1].get_lm_usage() is not None
+
+ lm_usage0 = results[0].get_lm_usage()["openai/gpt-4o-mini"]
+ lm_usage1 = results[1].get_lm_usage()["openai/gpt-4o-mini"]
+ assert lm_usage0["prompt_tokens"] == 1117
+ assert lm_usage1["prompt_tokens"] == 1117
+ assert lm_usage0["completion_tokens"] == 46
+ assert lm_usage1["completion_tokens"] == 46
+ assert lm_usage0["total_tokens"] == 1163
+ assert lm_usage1["total_tokens"] == 1163
+
+
def test_module_history():
class MyProgram(dspy.Module):
def __init__(self, **kwargs):
@@ -404,12 +454,12 @@ async def test_module_history_async():
],
model="openai/gpt-4o-mini",
)
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
program = MyProgram()
- await program.acall(question="What is the capital of France?")
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+ await program.acall(question="What is the capital of France?")
- # Second call only call the submodule.
- await program.cot.acall(question="What is the capital of France?")
+ # Second call only call the submodule.
+ await program.cot.acall(question="What is the capital of France?")
# The LM history entity exists in all the ancestor callers.
assert len(program.history) == 1
@@ -421,17 +471,20 @@ async def test_module_history_async():
assert program.history[0]["outputs"] == ["{'reasoning': 'Paris is the captial of France', 'answer': 'Paris'}"]
- dspy.settings.configure(disable_history=True)
+ with dspy.context(
+ disable_history=True, lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()
+ ):
+ await program.acall(question="What is the capital of France?")
- await program.acall(question="What is the capital of France?")
# No history is recorded when history is disabled.
assert len(program.history) == 1
assert len(program.cot.history) == 2
assert len(program.cot.predict.history) == 2
- dspy.settings.configure(disable_history=False)
-
- await program.acall(question="What is the capital of France?")
+ with dspy.context(
+ disable_history=False, lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()
+ ):
+ await program.acall(question="What is the capital of France?")
# History is recorded again when history is enabled.
assert len(program.history) == 2
assert len(program.cot.history) == 3
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 6019964c1..61007f427 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -142,8 +142,6 @@ async def test_stream_listener_chat_adapter():
judgement = self.predict2(question=x, answer=answer, **kwargs)
return judgement
- # Turn off the cache to ensure the stream is produced.
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
my_program = MyProgram()
program = dspy.streamify(
my_program,
@@ -153,11 +151,13 @@ async def test_stream_listener_chat_adapter():
],
include_final_prediction_in_output_stream=False,
)
- output = program(x="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ # Turn off the cache to ensure the stream is produced.
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):
+ output = program(x="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -205,8 +205,6 @@ async def test_stream_listener_json_adapter():
judgement = self.predict2(question=x, answer=answer, **kwargs)
return judgement
- # Turn off the cache to ensure the stream is produced.
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
my_program = MyProgram()
program = dspy.streamify(
my_program,
@@ -216,11 +214,13 @@ async def test_stream_listener_json_adapter():
],
include_final_prediction_in_output_stream=False,
)
- output = program(x="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ # Turn off the cache to ensure the stream is produced.
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+ output = program(x="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -241,8 +241,6 @@ def test_sync_streaming():
judgement = self.predict2(question=x, answer=answer, **kwargs)
return judgement
- # Turn off the cache to ensure the stream is produced.
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
my_program = MyProgram()
program = dspy.streamify(
my_program,
@@ -253,11 +251,13 @@ def test_sync_streaming():
include_final_prediction_in_output_stream=False,
async_streaming=False,
)
- output = program(x="why did a chicken cross the kitchen?")
- all_chunks = []
- for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ # Turn off the cache to ensure the stream is produced.
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):
+ output = program(x="why did a chicken cross the kitchen?")
+ all_chunks = []
+ for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -351,8 +351,6 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():
yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])
yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
-
stream_generators = [gpt_4o_mini_stream_1, gpt_4o_mini_stream_2]
async def completion_side_effect(*args, **kwargs):
@@ -366,11 +364,12 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():
dspy.streaming.StreamListener(signature_field_name="judgement"),
],
)
- output = program(question="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -406,8 +405,6 @@ async def test_stream_listener_returns_correct_chunk_json_adapter():
judgement = self.predict2(question=question, answer=answer, **kwargs)
return judgement
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
-
async def gpt_4o_mini_stream_1(*args, **kwargs):
yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])
yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="answer"))])
@@ -463,11 +460,12 @@ async def test_stream_listener_returns_correct_chunk_json_adapter():
dspy.streaming.StreamListener(signature_field_name="judgement"),
],
)
- output = program(question="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -499,8 +497,6 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_st
judgement = self.predict2(question=question, answer=answer, **kwargs)
return judgement
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter())
-
async def gemini_stream_1(*args, **kwargs):
yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])
yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content=" answer ## ]]"))])
@@ -542,11 +538,12 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_st
dspy.streaming.StreamListener(signature_field_name="judgement"),
],
)
- output = program(question="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ with dspy.context(lm=dspy.LM("gemini/gemini-2.5-flash", cache=False), adapter=dspy.ChatAdapter()):
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
@@ -573,8 +570,6 @@ async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_st
judgement = self.predict2(question=question, answer=answer, **kwargs)
return judgement
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
-
async def gemini_stream_1(*args, **kwargs):
yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])
yield ModelResponseStream(
@@ -610,11 +605,12 @@ async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_st
dspy.streaming.StreamListener(signature_field_name="judgement"),
],
)
- output = program(question="why did a chicken cross the kitchen?")
- all_chunks = []
- async for value in output:
- if isinstance(value, dspy.streaming.StreamResponse):
- all_chunks.append(value)
+ with dspy.context(lm=dspy.LM("gemini/gemini-2.5-flash", cache=False), adapter=dspy.JSONAdapter()):
+ output = program(question="why did a chicken cross the kitchen?")
+ all_chunks = []
+ async for value in output:
+ if isinstance(value, dspy.streaming.StreamResponse):
+ all_chunks.append(value)
assert all_chunks[0].predict_name == "predict1"
assert all_chunks[0].signature_field_name == "answer"
diff --git a/tests/utils/test_asyncify.py b/tests/utils/test_asyncify.py
index ffe959042..d34209ad6 100644
--- a/tests/utils/test_asyncify.py
+++ b/tests/utils/test_asyncify.py
@@ -14,10 +14,10 @@ async def test_async_limiter():
assert limiter.total_tokens == 8, "Default async capacity should be 8"
assert get_limiter() == limiter, "AsyncLimiter should be a singleton"
- dspy.settings.configure(async_max_workers=16)
- assert get_limiter() == limiter, "AsyncLimiter should be a singleton"
- assert get_limiter().total_tokens == 16, "Async capacity should be 16"
- assert get_limiter() == get_limiter(), "AsyncLimiter should be a singleton"
+ with dspy.context(async_max_workers=16):
+ assert get_limiter() == limiter, "AsyncLimiter should be a singleton"
+ assert get_limiter().total_tokens == 16, "Async capacity should be 16"
+ assert get_limiter() == get_limiter(), "AsyncLimiter should be a singleton"
@pytest.mark.anyio
@@ -32,12 +32,11 @@ async def test_asyncify():
await asyncio.gather(*[ask_the_question(wait) for _ in range(n)])
async def verify_asyncify(capacity: int, number_of_tasks: int, wait: float = 0.5):
- dspy.settings.configure(async_max_workers=capacity)
-
- start = time()
- await run_n_tasks(number_of_tasks, wait)
- end = time()
- total_time = end - start
+ with dspy.context(async_max_workers=capacity):
+ start = time()
+ await run_n_tasks(number_of_tasks, wait)
+ end = time()
+ total_time = end - start
# If asyncify is working correctly, the total time should be less than the total number of loops
# `(number_of_tasks / capacity)` times wait time, plus the computational overhead. The lower bound should
diff --git a/tests/utils/test_settings.py b/tests/utils/test_settings.py
new file mode 100644
index 000000000..590420a14
--- /dev/null
+++ b/tests/utils/test_settings.py
@@ -0,0 +1,167 @@
+import asyncio
+import time
+from concurrent.futures import ThreadPoolExecutor
+from unittest import mock
+
+import pytest
+from litellm import Choices, Message, ModelResponse
+
+import dspy
+
+
+def test_basic_dspy_settings():
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+ assert dspy.settings.lm.model == "openai/gpt-4o"
+ assert isinstance(dspy.settings.adapter, dspy.JSONAdapter)
+ assert len(dspy.settings.callbacks) == 1
+
+
+def test_forbid_configure_call_in_child_thread():
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+
+ def worker():
+ with pytest.raises(RuntimeError, match="Cannot call dspy.configure in a child thread"):
+ dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[])
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ executor.submit(worker)
+
+
+async def test_forbid_configure_call_in_async_function():
+ with pytest.raises(
+ RuntimeError,
+ match=r"dspy.settings.configure\(\.\.\.\) cannot be called a second time from*",
+ ):
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[]):
+ # context is allowed
+ pass
+
+
+def test_dspy_context():
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[]):
+ assert dspy.settings.lm.model == "openai/gpt-4o-mini"
+ assert len(dspy.settings.callbacks) == 0
+
+ assert dspy.settings.lm.model == "openai/gpt-4o"
+ assert len(dspy.settings.callbacks) == 1
+
+
+def test_dspy_context_parallel():
+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
+
+ def worker(i):
+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), trace=[i], callbacks=[]):
+ assert dspy.settings.lm.model == "openai/gpt-4o-mini"
+ assert dspy.settings.trace == [i]
+ assert len(dspy.settings.callbacks) == 0
+
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ executor.map(worker, range(3))
+
+ assert dspy.settings.lm.model == "openai/gpt-4o"
+ assert len(dspy.settings.callbacks) == 1
+
+
+def test_dspy_context_with_dspy_parallel():
+ dspy.configure(lm=dspy.LM("openai/gpt-4o", cache=False), adapter=dspy.ChatAdapter())
+
+ class MyModule(dspy.Module):
+ def __init__(self):
+ self.predict = dspy.Predict("question -> answer")
+
+ def forward(self, question: str) -> str:
+ lm = dspy.LM("openai/gpt-4o-mini", cache=False) if "France" in question else dspy.settings.lm
+ with dspy.context(lm=lm):
+ time.sleep(1)
+ assert dspy.settings.lm.model == lm.model
+ return self.predict(question=question)
+
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],
+ model="openai/gpt-4o-mini",
+ )
+
+ module = MyModule()
+ parallelizer = dspy.Parallel()
+ input_pairs = [
+ (module, {"question": "What is the capital of France?"}),
+ (module, {"question": "What is the capital of Germany?"}),
+ ]
+ parallelizer(input_pairs)
+
+ # Verify mock was called correctly
+ assert mock_completion.call_count == 2
+ for call_args in mock_completion.call_args_list:
+ if "France" in call_args.kwargs["messages"][-1]["content"]:
+ # France question uses gpt-4o-mini
+ assert call_args.kwargs["model"] == "openai/gpt-4o-mini"
+ else:
+ # Germany question uses gpt-4o
+ assert call_args.kwargs["model"] == "openai/gpt-4o"
+
+ # The main thread is not affected by the context
+ assert dspy.settings.lm.model == "openai/gpt-4o"
+
+
+async def test_dspy_context_with_async_task_group():
+ class MyModule(dspy.Module):
+ def __init__(self):
+ self.predict = dspy.Predict("question -> answer")
+
+ async def aforward(self, question: str) -> str:
+ lm = (
+ dspy.LM("openai/gpt-4o-mini", cache=False)
+ if "France" in question
+ else dspy.LM("openai/gpt-4o", cache=False)
+ )
+ with dspy.context(lm=lm, trace=[]):
+ await asyncio.sleep(1)
+ assert dspy.settings.lm.model == lm.model
+ result = await self.predict.acall(question=question)
+ assert len(dspy.settings.trace) == 1
+ return result
+
+ module = MyModule()
+
+ with dspy.context(lm=dspy.LM("openai/gpt-4.1", cache=False), adapter=dspy.ChatAdapter()):
+ with mock.patch("litellm.acompletion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],
+ model="openai/gpt-4o-mini",
+ )
+
+ # Define the coroutines to be run
+ coroutines = [
+ module.acall(question="What is the capital of France?"),
+ module.acall(question="What is the capital of France?"),
+ module.acall(question="What is the capital of Germany?"),
+ module.acall(question="What is the capital of Germany?"),
+ ]
+
+ # Run them concurrently and gather results
+ results = await asyncio.gather(*coroutines)
+
+ assert results[0].answer == "Paris"
+ assert results[1].answer == "Paris"
+ assert results[2].answer == "Paris"
+ assert results[3].answer == "Paris"
+
+ # Verify mock was called correctly
+ assert mock_completion.call_count == 4
+ # France question uses gpt-4o-mini
+ assert mock_completion.call_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"
+ assert mock_completion.call_args_list[1].kwargs["model"] == "openai/gpt-4o-mini"
+ # Germany question uses gpt-4o
+ assert mock_completion.call_args_list[2].kwargs["model"] == "openai/gpt-4o"
+ assert mock_completion.call_args_list[3].kwargs["model"] == "openai/gpt-4o"
+
+ # The main thread is not affected by the context
+ assert dspy.settings.lm.model == "openai/gpt-4.1"
+ assert dspy.settings.trace == []