DSPy: The framework for programming—not prompting—language models
Python
Merge async settings change into main (#8361)
Chen Qiandd8cf1cb284d
diff --git a/dspy/dsp/utils/settings.py b/dspy/dsp/utils/settings.pyindex fc0f7300d..cf1c7d267 100644--- a/dspy/dsp/utils/settings.py+++ b/dspy/dsp/utils/settings.py@@ -1,3 +1,5 @@+import asyncio+import contextvarsimport copyimport threadingfrom contextlib import contextmanager@@ -38,13 +40,7 @@ config_owner_thread_id = None# Global lock for settings configurationglobal_lock = threading.Lock()--class ThreadLocalOverrides(threading.local):- def __init__(self):- self.overrides = dotdict()---thread_local_overrides = ThreadLocalOverrides()+thread_local_overrides = contextvars.ContextVar("context_overrides", default=dotdict())class Settings:@@ -75,7 +71,7 @@ class Settings:return global_lockdef __getattr__(self, name):- overrides = getattr(thread_local_overrides, "overrides", dotdict())+ overrides = thread_local_overrides.get()if name in overrides:return overrides[name]elif name in main_thread_config:@@ -96,7 +92,7 @@ class Settings:self.__setattr__(key, value)def __contains__(self, key):- overrides = getattr(thread_local_overrides, "overrides", dotdict())+ overrides = thread_local_overrides.get()return key in overrides or key in main_thread_configdef get(self, key, default=None):@@ -106,23 +102,60 @@ class Settings:return defaultdef copy(self):- overrides = getattr(thread_local_overrides, "overrides", dotdict())+ overrides = thread_local_overrides.get()return dotdict({**main_thread_config, **overrides})@propertydef config(self):return self.copy()- def configure(self, **kwargs):+ def _ensure_configure_allowed(self):global main_thread_config, config_owner_thread_idcurrent_thread_id = threading.get_ident()- with self.lock:- # First configuration: establish ownership. If ownership established, only that thread can configure.- if config_owner_thread_id in [None, current_thread_id]:- config_owner_thread_id = current_thread_id- else:- raise RuntimeError("dspy.settings can only be changed by the thread that initially configured it.")+ if config_owner_thread_id is None:+ # First `configure` call is always allowed.+ config_owner_thread_id = current_thread_id+ return++ if config_owner_thread_id != current_thread_id:+ # Disallow a second `configure` calls from other threads.+ raise RuntimeError("dspy.settings can only be changed by the thread that initially configured it.")++ # Async task doesn't allow a second `configure` call, must use dspy.context(...) instead.+ is_async_task = False+ try:+ if asyncio.current_task() is not None:+ is_async_task = True+ except RuntimeError:+ # This exception (e.g., "no current task") means we are not in an async loop/task,+ # or asyncio module itself is not fully functional in this specific sub-thread context.+ is_async_task = False++ if not is_async_task:+ return++ # We are in an async task. Now check for IPython and allow calling `configure` from IPython.+ in_ipython = False+ try:+ from IPython import get_ipython++ # get_ipython is a global injected by IPython environments.+ # We check its existence and type to be more robust.+ in_ipython = get_ipython() is not None+ except Exception:+ # If `IPython` is not installed or `get_ipython` failed, we are not in an IPython environment.+ in_ipython = False++ if not in_ipython:+ raise RuntimeError(+ "dspy.settings.configure(...) cannot be called a second time from an async task. Use "+ "`dspy.context(...)` instead."+ )++ def configure(self, **kwargs):+ # If no exception is raised, the `configure` call is allowed.+ self._ensure_configure_allowed()# Update global configfor k, v in kwargs.items():@@ -136,17 +169,17 @@ class Settings:If threads are spawned inside this block using ParallelExecutor, they will inherit these overrides."""- original_overrides = getattr(thread_local_overrides, "overrides", dotdict()).copy()+ original_overrides = thread_local_overrides.get().copy()new_overrides = dotdict({**main_thread_config, **original_overrides, **kwargs})- thread_local_overrides.overrides = new_overrides+ token = thread_local_overrides.set(new_overrides)try:yieldfinally:- thread_local_overrides.overrides = original_overrides+ thread_local_overrides.reset(token)def __repr__(self):- overrides = getattr(thread_local_overrides, "overrides", dotdict())+ overrides = thread_local_overrides.get()combined_config = {**main_thread_config, **overrides}return repr(combined_config)diff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex f39cc8912..e306591e8 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -2,7 +2,7 @@ from typing import Optionalimport magicattr-from dspy.dsp.utils.settings import settings+from dspy.dsp.utils.settings import settings, thread_local_overridesfrom dspy.predict.parallel import Parallelfrom dspy.primitives.module import BaseModulefrom dspy.utils.callback import with_callbacks@@ -51,7 +51,7 @@ class Module(BaseModule, metaclass=ProgramMeta):caller_modules.append(self)with settings.context(caller_modules=caller_modules):- if settings.track_usage and settings.usage_tracker is None:+ if settings.track_usage and thread_local_overrides.get().get("usage_tracker") is None:with track_usage() as usage_tracker:output = self.forward(*args, **kwargs)output.set_lm_usage(usage_tracker.get_total_tokens())@@ -66,7 +66,7 @@ class Module(BaseModule, metaclass=ProgramMeta):caller_modules.append(self)with settings.context(caller_modules=caller_modules):- if settings.track_usage and settings.usage_tracker is None:+ if settings.track_usage and thread_local_overrides.get().get("usage_tracker") is None:with track_usage() as usage_tracker:output = await self.aforward(*args, **kwargs)output.set_lm_usage(usage_tracker.get_total_tokens())diff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.pyindex 98adeacf4..534f60fcc 100644--- a/dspy/streaming/streamify.py+++ b/dspy/streaming/streamify.py@@ -222,16 +222,10 @@ def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:# To propagate prediction request ID context to the child threadcontext = contextvars.copy_context()- from dspy.dsp.utils.settings import thread_local_overrides-- parent_overrides = thread_local_overrides.overrides.copy()def producer():"""Runs in a background thread to fetch items asynchronously."""- original_overrides = thread_local_overrides.overrides- thread_local_overrides.overrides = parent_overrides.copy()-async def runner():try:async for item in async_generator:@@ -241,7 +235,6 @@ def apply_sync_streaming(async_generator: AsyncGenerator) -> Generator:queue.put(stop_sentinel)context.run(asyncio.run, runner())- thread_local_overrides.overrides = original_overrides# Start the producer in a background threadthread = threading.Thread(target=producer, daemon=True)diff --git a/dspy/utils/asyncify.py b/dspy/utils/asyncify.pyindex 7596a03f1..5ca9cdf32 100644--- a/dspy/utils/asyncify.py+++ b/dspy/utils/asyncify.py@@ -46,17 +46,17 @@ def asyncify(program: "Module") -> Callable[[Any, Any], Awaitable[Any]]:# Capture the current overrides at call-time.from dspy.dsp.utils.settings import thread_local_overrides- parent_overrides = thread_local_overrides.overrides.copy()+ parent_overrides = thread_local_overrides.get().copy()def wrapped_program(*a, **kw):from dspy.dsp.utils.settings import thread_local_overrides- original_overrides = thread_local_overrides.overrides- thread_local_overrides.overrides = parent_overrides.copy()+ original_overrides = thread_local_overrides.get()+ token = thread_local_overrides.set({**original_overrides, **parent_overrides.copy()})try:return program(*a, **kw)finally:- thread_local_overrides.overrides = original_overrides+ thread_local_overrides.reset(token)# Create a fresh asyncified callable each time, ensuring the latest context is used.call_async = asyncer.asyncify(wrapped_program, abandon_on_cancel=True, limiter=get_limiter())diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.pyindex 9ad76493f..d45df534e 100644--- a/dspy/utils/parallelizer.py+++ b/dspy/utils/parallelizer.py@@ -86,8 +86,8 @@ class ParallelExecutor:# Apply parent's thread-local overridesfrom dspy.dsp.utils.settings import thread_local_overrides- original = thread_local_overrides.overrides- thread_local_overrides.overrides = parent_overrides.copy()+ original = thread_local_overrides.get()+ token = thread_local_overrides.set({**original, **parent_overrides.copy()})if parent_overrides.get("usage_tracker"):# Usage tracker needs to be deep copied across threads so that each thread tracks its own usagethread_local_overrides.overrides["usage_tracker"] = copy.deepcopy(parent_overrides["usage_tracker"])@@ -95,7 +95,7 @@ class ParallelExecutor:try:return index, function(item)finally:- thread_local_overrides.overrides = original+ thread_local_overrides.reset(token)# Handle Ctrl-C in the main thread@contextlib.contextmanager@@ -121,7 +121,7 @@ class ParallelExecutor:with interrupt_manager():from dspy.dsp.utils.settings import thread_local_overrides- parent_overrides = thread_local_overrides.overrides.copy()+ parent_overrides = thread_local_overrides.get().copy()futures_map = {}futures_set = set()diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.pyindex c7a9460a3..f045188fb 100644--- a/tests/adapters/test_two_step_adapter.py+++ b/tests/adapters/test_two_step_adapter.py@@ -94,9 +94,8 @@ async def test_two_step_adapter_async_call():mock_extraction_lm.kwargs = {"temperature": 1.0}mock_extraction_lm.model = "openai/gpt-4o"- dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))-- result = await program.acall(question="What is 5 + 7?")+ with dspy.context(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm)):+ result = await program.acall(question="What is 5 + 7?")assert result.answer == 12diff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.pyindex b3df2b0cd..365266ca9 100644--- a/tests/callback/test_callback.py+++ b/tests/callback/test_callback.py@@ -189,13 +189,12 @@ def test_callback_complex_module():@pytest.mark.asyncioasync def test_callback_async_module():callback = MyCallback()- dspy.settings.configure(+ with dspy.context(lm=DummyLM({"How are you?": {"answer": "test output", "reasoning": "No more responses"}}),callbacks=[callback],- )-- cot = dspy.ChainOfThought("question -> answer", n=3)- result = await cot.acall(question="How are you?")+ ):+ cot = dspy.ChainOfThought("question -> answer", n=3)+ result = await cot.acall(question="How are you?")assert result["answer"] == "test output"assert result["reasoning"] == "No more responses"diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pyindex 68f8e4c9b..247256dba 100644--- a/tests/predict/test_chain_of_thought.py+++ b/tests/predict/test_chain_of_thought.py@@ -19,7 +19,7 @@ def test_initialization_with_string_signature():@pytest.mark.asyncioasync def test_async_chain_of_thought():lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])- dspy.settings.configure(lm=lm)- program = ChainOfThought("question -> answer")- result = await program.acall(question="What is 1+1?")- assert result.answer == "2"+ with dspy.context(lm=lm):+ program = ChainOfThought("question -> answer")+ result = await program.acall(question="What is 1+1?")+ assert result.answer == "2"diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.pyindex d4ce96158..c1aea9727 100644--- a/tests/predict/test_predict.py+++ b/tests/predict/test_predict.py@@ -1,5 +1,8 @@+import asyncioimport copyimport enum+import time+import typesfrom datetime import datetimefrom unittest.mock import patch@@ -506,6 +509,69 @@ def test_lm_usage():assert result.get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+def test_lm_usage_with_parallel():+ program = Predict("question -> answer")++ def program_wrapper(question):+ # Sleep to make it possible to cause a race condition+ time.sleep(0.5)+ return program(question=question)++ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True)+ with patch(+ "dspy.clients.lm.litellm_completion",+ return_value=ModelResponse(+ choices=[{"message": {"content": "[[ ## answer ## ]]\nParis"}}],+ usage={"total_tokens": 10},+ ),+ ):+ parallelizer = dspy.Parallel()+ input_pairs = [+ (program_wrapper, {"question": "What is the capital of France?"}),+ (program_wrapper, {"question": "What is the capital of France?"}),+ ]+ results = parallelizer(input_pairs)+ assert results[0].answer == "Paris"+ assert results[1].answer == "Paris"+ assert results[0].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+ assert results[1].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+++async def test_lm_usage_with_async():+ program = Predict("question -> answer")++ original_aforward = program.aforward++ async def patched_aforward(self, **kwargs):+ await asyncio.sleep(1)+ return await original_aforward(**kwargs)++ program.aforward = types.MethodType(patched_aforward, program)++ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True):+ with patch(+ "litellm.acompletion",+ return_value=ModelResponse(+ choices=[{"message": {"content": "[[ ## answer ## ]]\nParis"}}],+ usage={"total_tokens": 10},+ ),+ ):+ coroutines = [+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ ]+ results = await asyncio.gather(*coroutines)+ assert results[0].answer == "Paris"+ assert results[1].answer == "Paris"+ assert results[0].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+ assert results[1].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+ assert results[2].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10+ assert results[3].get_lm_usage()["openai/gpt-4o-mini"]["total_tokens"] == 10++def test_positional_arguments():program = Predict("question -> answer")with pytest.raises(ValueError) as e:@@ -569,9 +635,9 @@ def test_field_constraints(adapter_type):@pytest.mark.asyncioasync def test_async_predict():program = Predict("question -> answer")- dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]))- result = await program.acall(question="What is the capital of France?")- assert result.answer == "Paris"+ with dspy.context(lm=DummyLM([{"answer": "Paris"}])):+ result = await program.acall(question="What is the capital of France?")+ assert result.answer == "Paris"def test_predicted_outputs_piped_from_predict_to_lm_call():diff --git a/tests/predict/test_react.py b/tests/predict/test_react.pyindex e3beb300d..4ae1b6f52 100644--- a/tests/predict/test_react.py+++ b/tests/predict/test_react.py@@ -254,16 +254,15 @@ async def test_async_tool_calling_with_pydantic_args():},])- dspy.settings.configure(lm=lm)-- outputs = await react.acall(- participant_name="Alice",- event_info=CalendarEvent(- name="Science Fair",- date="Friday",- participants={"Alice": "female", "Bob": "male"},- ),- )+ with dspy.context(lm=lm):+ outputs = await react.acall(+ participant_name="Alice",+ event_info=CalendarEvent(+ name="Science Fair",+ date="Friday",+ participants={"Alice": "female", "Bob": "male"},+ ),+ )assert outputs.invitation_letter == "It's my honor to invite Alice to the Science Fair event on Friday."expected_trajectory = {@@ -309,9 +308,8 @@ async def test_async_error_retry():{"reasoning": "I added the numbers successfully", "c": 3},])- dspy.settings.configure(lm=lm)-- outputs = await react.acall(a=1, b=2, max_iters=2)+ with dspy.context(lm=lm):+ outputs = await react.acall(a=1, b=2, max_iters=2)traj = outputs.trajectory# Exact-match checks (thoughts + tool calls)diff --git a/tests/primitives/test_module.py b/tests/primitives/test_module.pyindex bc1d2975c..544146839 100644--- a/tests/primitives/test_module.py+++ b/tests/primitives/test_module.py@@ -1,3 +1,4 @@+import asyncioimport loggingimport osimport threading@@ -5,6 +6,7 @@ from unittest.mock import patchimport pytestfrom litellm import Choices, Message, ModelResponse+from litellm.types.utils import Usageimport dspyfrom dspy.utils.dummies import DummyLM@@ -307,6 +309,54 @@ def test_usage_tracker_in_parallel():assert results[1].get_lm_usage().keys() == set(["openai/gpt-3.5-turbo"])+async def test_usage_tracker_async_parallel():+ program = dspy.Predict("question -> answer")++ with patch("litellm.acompletion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],+ usage=Usage(+ **{+ "prompt_tokens": 1117,+ "completion_tokens": 46,+ "total_tokens": 1163,+ "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},+ "completion_tokens_details": {+ "reasoning_tokens": 0,+ "audio_tokens": 0,+ "accepted_prediction_tokens": 0,+ "rejected_prediction_tokens": 0,+ },+ },+ ),+ model="openai/gpt-4o-mini",+ )++ coroutines = [+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ program.acall(question="What is the capital of France?"),+ ]+ with dspy.settings.context(+ lm=dspy.LM("openai/gpt-4o-mini", cache=False), track_usage=True, adapter=dspy.JSONAdapter()+ ):+ results = await asyncio.gather(*coroutines)++ assert results[0].get_lm_usage() is not None+ assert results[1].get_lm_usage() is not None++ lm_usage0 = results[0].get_lm_usage()["openai/gpt-4o-mini"]+ lm_usage1 = results[1].get_lm_usage()["openai/gpt-4o-mini"]+ assert lm_usage0["prompt_tokens"] == 1117+ assert lm_usage1["prompt_tokens"] == 1117+ assert lm_usage0["completion_tokens"] == 46+ assert lm_usage1["completion_tokens"] == 46+ assert lm_usage0["total_tokens"] == 1163+ assert lm_usage1["total_tokens"] == 1163++def test_module_history():class MyProgram(dspy.Module):def __init__(self, **kwargs):@@ -404,12 +454,12 @@ async def test_module_history_async():],model="openai/gpt-4o-mini",)- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())program = MyProgram()- await program.acall(question="What is the capital of France?")+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):+ await program.acall(question="What is the capital of France?")- # Second call only call the submodule.- await program.cot.acall(question="What is the capital of France?")+ # Second call only call the submodule.+ await program.cot.acall(question="What is the capital of France?")# The LM history entity exists in all the ancestor callers.assert len(program.history) == 1@@ -421,17 +471,20 @@ async def test_module_history_async():assert program.history[0]["outputs"] == ["{'reasoning': 'Paris is the captial of France', 'answer': 'Paris'}"]- dspy.settings.configure(disable_history=True)+ with dspy.context(+ disable_history=True, lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()+ ):+ await program.acall(question="What is the capital of France?")- await program.acall(question="What is the capital of France?")# No history is recorded when history is disabled.assert len(program.history) == 1assert len(program.cot.history) == 2assert len(program.cot.predict.history) == 2- dspy.settings.configure(disable_history=False)-- await program.acall(question="What is the capital of France?")+ with dspy.context(+ disable_history=False, lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()+ ):+ await program.acall(question="What is the capital of France?")# History is recorded again when history is enabled.assert len(program.history) == 2assert len(program.cot.history) == 3diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 6019964c1..61007f427 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -142,8 +142,6 @@ async def test_stream_listener_chat_adapter():judgement = self.predict2(question=x, answer=answer, **kwargs)return judgement- # Turn off the cache to ensure the stream is produced.- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))my_program = MyProgram()program = dspy.streamify(my_program,@@ -153,11 +151,13 @@ async def test_stream_listener_chat_adapter():],include_final_prediction_in_output_stream=False,)- output = program(x="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ # Turn off the cache to ensure the stream is produced.+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):+ output = program(x="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -205,8 +205,6 @@ async def test_stream_listener_json_adapter():judgement = self.predict2(question=x, answer=answer, **kwargs)return judgement- # Turn off the cache to ensure the stream is produced.- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())my_program = MyProgram()program = dspy.streamify(my_program,@@ -216,11 +214,13 @@ async def test_stream_listener_json_adapter():],include_final_prediction_in_output_stream=False,)- output = program(x="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ # Turn off the cache to ensure the stream is produced.+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):+ output = program(x="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -241,8 +241,6 @@ def test_sync_streaming():judgement = self.predict2(question=x, answer=answer, **kwargs)return judgement- # Turn off the cache to ensure the stream is produced.- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))my_program = MyProgram()program = dspy.streamify(my_program,@@ -253,11 +251,13 @@ def test_sync_streaming():include_final_prediction_in_output_stream=False,async_streaming=False,)- output = program(x="why did a chicken cross the kitchen?")- all_chunks = []- for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ # Turn off the cache to ensure the stream is produced.+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):+ output = program(x="why did a chicken cross the kitchen?")+ all_chunks = []+ for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -351,8 +351,6 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ##"))])yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))])- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))-stream_generators = [gpt_4o_mini_stream_1, gpt_4o_mini_stream_2]async def completion_side_effect(*args, **kwargs):@@ -366,11 +364,12 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter():dspy.streaming.StreamListener(signature_field_name="judgement"),],)- output = program(question="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False)):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -406,8 +405,6 @@ async def test_stream_listener_returns_correct_chunk_json_adapter():judgement = self.predict2(question=question, answer=answer, **kwargs)return judgement- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())-async def gpt_4o_mini_stream_1(*args, **kwargs):yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content='{"'))])yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="answer"))])@@ -463,11 +460,12 @@ async def test_stream_listener_returns_correct_chunk_json_adapter():dspy.streaming.StreamListener(signature_field_name="judgement"),],)- output = program(question="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -499,8 +497,6 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_stjudgement = self.predict2(question=question, answer=answer, **kwargs)return judgement- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter())-async def gemini_stream_1(*args, **kwargs):yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="[[ ##"))])yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content=" answer ## ]]"))])@@ -542,11 +538,12 @@ async def test_stream_listener_returns_correct_chunk_chat_adapter_untokenized_stdspy.streaming.StreamListener(signature_field_name="judgement"),],)- output = program(question="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ with dspy.context(lm=dspy.LM("gemini/gemini-2.5-flash", cache=False), adapter=dspy.ChatAdapter()):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"@@ -573,8 +570,6 @@ async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_stjudgement = self.predict2(question=question, answer=answer, **kwargs)return judgement- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())-async def gemini_stream_1(*args, **kwargs):yield ModelResponseStream(model="gemini", choices=[StreamingChoices(delta=Delta(content="{\n"))])yield ModelResponseStream(@@ -610,11 +605,12 @@ async def test_stream_listener_returns_correct_chunk_json_adapter_untokenized_stdspy.streaming.StreamListener(signature_field_name="judgement"),],)- output = program(question="why did a chicken cross the kitchen?")- all_chunks = []- async for value in output:- if isinstance(value, dspy.streaming.StreamResponse):- all_chunks.append(value)+ with dspy.context(lm=dspy.LM("gemini/gemini-2.5-flash", cache=False), adapter=dspy.JSONAdapter()):+ output = program(question="why did a chicken cross the kitchen?")+ all_chunks = []+ async for value in output:+ if isinstance(value, dspy.streaming.StreamResponse):+ all_chunks.append(value)assert all_chunks[0].predict_name == "predict1"assert all_chunks[0].signature_field_name == "answer"diff --git a/tests/utils/test_asyncify.py b/tests/utils/test_asyncify.pyindex ffe959042..d34209ad6 100644--- a/tests/utils/test_asyncify.py+++ b/tests/utils/test_asyncify.py@@ -14,10 +14,10 @@ async def test_async_limiter():assert limiter.total_tokens == 8, "Default async capacity should be 8"assert get_limiter() == limiter, "AsyncLimiter should be a singleton"- dspy.settings.configure(async_max_workers=16)- assert get_limiter() == limiter, "AsyncLimiter should be a singleton"- assert get_limiter().total_tokens == 16, "Async capacity should be 16"- assert get_limiter() == get_limiter(), "AsyncLimiter should be a singleton"+ with dspy.context(async_max_workers=16):+ assert get_limiter() == limiter, "AsyncLimiter should be a singleton"+ assert get_limiter().total_tokens == 16, "Async capacity should be 16"+ assert get_limiter() == get_limiter(), "AsyncLimiter should be a singleton"@pytest.mark.anyio@@ -32,12 +32,11 @@ async def test_asyncify():await asyncio.gather(*[ask_the_question(wait) for _ in range(n)])async def verify_asyncify(capacity: int, number_of_tasks: int, wait: float = 0.5):- dspy.settings.configure(async_max_workers=capacity)-- start = time()- await run_n_tasks(number_of_tasks, wait)- end = time()- total_time = end - start+ with dspy.context(async_max_workers=capacity):+ start = time()+ await run_n_tasks(number_of_tasks, wait)+ end = time()+ total_time = end - start# If asyncify is working correctly, the total time should be less than the total number of loops# `(number_of_tasks / capacity)` times wait time, plus the computational overhead. The lower bound shoulddiff --git a/tests/utils/test_settings.py b/tests/utils/test_settings.pynew file mode 100644index 000000000..590420a14--- /dev/null+++ b/tests/utils/test_settings.py@@ -0,0 +1,167 @@+import asyncio+import time+from concurrent.futures import ThreadPoolExecutor+from unittest import mock++import pytest+from litellm import Choices, Message, ModelResponse++import dspy+++def test_basic_dspy_settings():+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])+ assert dspy.settings.lm.model == "openai/gpt-4o"+ assert isinstance(dspy.settings.adapter, dspy.JSONAdapter)+ assert len(dspy.settings.callbacks) == 1+++def test_forbid_configure_call_in_child_thread():+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])++ def worker():+ with pytest.raises(RuntimeError, match="Cannot call dspy.configure in a child thread"):+ dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[])++ with ThreadPoolExecutor(max_workers=1) as executor:+ executor.submit(worker)+++async def test_forbid_configure_call_in_async_function():+ with pytest.raises(+ RuntimeError,+ match=r"dspy.settings.configure\(\.\.\.\) cannot be called a second time from*",+ ):+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])++ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[]):+ # context is allowed+ pass+++def test_dspy_context():+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[]):+ assert dspy.settings.lm.model == "openai/gpt-4o-mini"+ assert len(dspy.settings.callbacks) == 0++ assert dspy.settings.lm.model == "openai/gpt-4o"+ assert len(dspy.settings.callbacks) == 1+++def test_dspy_context_parallel():+ dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])++ def worker(i):+ with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), trace=[i], callbacks=[]):+ assert dspy.settings.lm.model == "openai/gpt-4o-mini"+ assert dspy.settings.trace == [i]+ assert len(dspy.settings.callbacks) == 0++ with ThreadPoolExecutor(max_workers=5) as executor:+ executor.map(worker, range(3))++ assert dspy.settings.lm.model == "openai/gpt-4o"+ assert len(dspy.settings.callbacks) == 1+++def test_dspy_context_with_dspy_parallel():+ dspy.configure(lm=dspy.LM("openai/gpt-4o", cache=False), adapter=dspy.ChatAdapter())++ class MyModule(dspy.Module):+ def __init__(self):+ self.predict = dspy.Predict("question -> answer")++ def forward(self, question: str) -> str:+ lm = dspy.LM("openai/gpt-4o-mini", cache=False) if "France" in question else dspy.settings.lm+ with dspy.context(lm=lm):+ time.sleep(1)+ assert dspy.settings.lm.model == lm.model+ return self.predict(question=question)++ with mock.patch("litellm.completion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],+ model="openai/gpt-4o-mini",+ )++ module = MyModule()+ parallelizer = dspy.Parallel()+ input_pairs = [+ (module, {"question": "What is the capital of France?"}),+ (module, {"question": "What is the capital of Germany?"}),+ ]+ parallelizer(input_pairs)++ # Verify mock was called correctly+ assert mock_completion.call_count == 2+ for call_args in mock_completion.call_args_list:+ if "France" in call_args.kwargs["messages"][-1]["content"]:+ # France question uses gpt-4o-mini+ assert call_args.kwargs["model"] == "openai/gpt-4o-mini"+ else:+ # Germany question uses gpt-4o+ assert call_args.kwargs["model"] == "openai/gpt-4o"++ # The main thread is not affected by the context+ assert dspy.settings.lm.model == "openai/gpt-4o"+++async def test_dspy_context_with_async_task_group():+ class MyModule(dspy.Module):+ def __init__(self):+ self.predict = dspy.Predict("question -> answer")++ async def aforward(self, question: str) -> str:+ lm = (+ dspy.LM("openai/gpt-4o-mini", cache=False)+ if "France" in question+ else dspy.LM("openai/gpt-4o", cache=False)+ )+ with dspy.context(lm=lm, trace=[]):+ await asyncio.sleep(1)+ assert dspy.settings.lm.model == lm.model+ result = await self.predict.acall(question=question)+ assert len(dspy.settings.trace) == 1+ return result++ module = MyModule()++ with dspy.context(lm=dspy.LM("openai/gpt-4.1", cache=False), adapter=dspy.ChatAdapter()):+ with mock.patch("litellm.acompletion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],+ model="openai/gpt-4o-mini",+ )++ # Define the coroutines to be run+ coroutines = [+ module.acall(question="What is the capital of France?"),+ module.acall(question="What is the capital of France?"),+ module.acall(question="What is the capital of Germany?"),+ module.acall(question="What is the capital of Germany?"),+ ]++ # Run them concurrently and gather results+ results = await asyncio.gather(*coroutines)++ assert results[0].answer == "Paris"+ assert results[1].answer == "Paris"+ assert results[2].answer == "Paris"+ assert results[3].answer == "Paris"++ # Verify mock was called correctly+ assert mock_completion.call_count == 4+ # France question uses gpt-4o-mini+ assert mock_completion.call_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"+ assert mock_completion.call_args_list[1].kwargs["model"] == "openai/gpt-4o-mini"+ # Germany question uses gpt-4o+ assert mock_completion.call_args_list[2].kwargs["model"] == "openai/gpt-4o"+ assert mock_completion.call_args_list[3].kwargs["model"] == "openai/gpt-4o"++ # The main thread is not affected by the context+ assert dspy.settings.lm.model == "openai/gpt-4.1"+ assert dspy.settings.trace == []