Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix infinite growing trace and add tests (#8607)

Chen Qianfce513777142
diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py
index 5247dda71..c0550b28e 100644
--- a/dspy/clients/base_lm.py
+++ b/dspy/clients/base_lm.py
@@ -84,11 +84,9 @@ class BaseLM:
"response_model": response.model,
"model_type": self.model_type,
}
- self.history.append(entry)
- self.update_global_history(entry)
- caller_modules = settings.caller_modules or []
- for module in caller_modules:
- module.history.append(entry)
+
+ self.update_history(entry)
+
return outputs
@with_callbacks
@@ -139,15 +137,32 @@ class BaseLM:
def inspect_history(self, n: int = 1):
return pretty_print_history(self.history, n)
- def update_global_history(self, entry):
+ def update_history(self, entry):
if settings.disable_history:
return
+ # Global LM history
if len(GLOBAL_HISTORY) >= MAX_HISTORY_SIZE:
GLOBAL_HISTORY.pop(0)
GLOBAL_HISTORY.append(entry)
+ if settings.max_history_size == 0:
+ return
+
+ # dspy.LM.history
+ if len(self.history) >= settings.max_history_size:
+ self.history.pop(0)
+
+ self.history.append(entry)
+
+ # Per-module history
+ caller_modules = settings.caller_modules or []
+ for module in caller_modules:
+ if len(module.history) >= settings.max_history_size:
+ module.history.pop(0)
+ module.history.append(entry)
+
def inspect_history(n: int = 1):
"""The global history shared across all LMs."""
diff --git a/dspy/dsp/utils/settings.py b/dspy/dsp/utils/settings.py
index eea41bcbe..f5319a9f9 100644
--- a/dspy/dsp/utils/settings.py
+++ b/dspy/dsp/utils/settings.py
@@ -26,6 +26,8 @@ DEFAULT_CONFIG = dotdict(
max_errors=10, # Maximum errors before halting operations.
# If true, async tools can be called in sync mode by getting converted to sync.
allow_tool_async_sync_conversion=False,
+ max_history_size=10000,
+ max_trace_size=10000,
)
# Global base configuration and owner tracking
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 59939b1ef..a893f0d66 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -147,8 +147,10 @@ class Predict(Module, Parameter):
def _forward_postprocess(self, completions, signature, **kwargs):
pred = Prediction.from_completions(completions, signature=signature)
- if kwargs.pop("_trace", True) and settings.trace is not None:
+ if kwargs.pop("_trace", True) and settings.trace is not None and settings.max_trace_size > 0:
trace = settings.trace
+ if len(trace) >= settings.max_trace_size:
+ trace.pop(0)
trace.append((self, {**kwargs}, pred))
return pred
diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py
index 631d618c3..6e83d1126 100644
--- a/dspy/utils/dummies.py
+++ b/dspy/utils/dummies.py
@@ -124,8 +124,7 @@ class DummyLM(LM):
entry = {"prompt": prompt, "messages": messages, "kwargs": kwargs}
entry = {**entry, "outputs": outputs, "usage": 0}
entry = {**entry, "cost": 0}
- self.history.append(entry)
- self.update_global_history(entry)
+ self.update_history(entry)
return outputs
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index b63406ba1..8aa4c89dc 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -374,3 +374,39 @@ async def test_async_lm_call_with_cache(tmp_path):
assert mock_alitellm_completion.call_count == 2
dspy.cache = original_cache
+
+
+def test_lm_history_size_limit():
+ lm = dspy.LM(model="openai/gpt-4o-mini")
+ with dspy.context(max_history_size=5):
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="test answer"))],
+ model="openai/gpt-4o-mini",
+ )
+
+ for _ in range(10):
+ lm("query")
+
+ assert len(lm.history) == 5
+
+
+def test_disable_history():
+ lm = dspy.LM(model="openai/gpt-4o-mini")
+ with dspy.context(disable_history=True):
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="test answer"))],
+ model="openai/gpt-4o-mini",
+ )
+ for _ in range(10):
+ lm("query")
+
+ assert len(lm.history) == 0
+
+ with dspy.context(disable_history=False):
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="test answer"))],
+ model="openai/gpt-4o-mini",
+ )
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index 37b9c87fc..31d80fc52 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -716,10 +716,7 @@ def test_dump_state_pydantic_non_primitive_types():
assert reloaded == serialized
predictor = Predict(TestSignature)
- demo = {
- "website_info": website_info,
- "summary": "This is a test website."
- }
+ demo = {"website_info": website_info, "summary": "This is a test website."}
predictor.demos = [demo]
state = predictor.dump_state()
@@ -729,3 +726,41 @@ def test_dump_state_pydantic_non_primitive_types():
demo_data = reloaded_state["demos"][0]
assert demo_data["website_info"]["url"] == "https://www.example.com/"
assert demo_data["website_info"]["created_at"] == "2021-01-01T12:00:00"
+
+
+def test_trace_size_limit():
+ program = Predict("question -> answer")
+ dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]), max_trace_size=3)
+
+ for _ in range(10):
+ program(question="What is the capital of France?")
+
+ assert len(dspy.settings.trace) == 3
+
+
+def test_disable_trace():
+ program = Predict("question -> answer")
+ dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]), trace=None)
+
+ for _ in range(10):
+ program(question="What is the capital of France?")
+
+ assert dspy.settings.trace is None
+
+
+def test_per_module_history_size_limit():
+ program = Predict("question -> answer")
+ dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]), max_history_size=5)
+
+ for _ in range(10):
+ program(question="What is the capital of France?")
+ assert len(program.history) == 5
+
+
+def test_per_module_history_disabled():
+ program = Predict("question -> answer")
+ dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]), disable_history=True)
+
+ for _ in range(10):
+ program(question="What is the capital of France?")
+ assert len(program.history) == 0