Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Limit callback metadata to trace capture path (#8835)

TomuHirata71fc252e1d0b
diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.py
index 5af24223c..2f0cc60cd 100644
--- a/dspy/teleprompt/bootstrap_trace.py
+++ b/dspy/teleprompt/bootstrap_trace.py
@@ -37,6 +37,7 @@ def bootstrap_trace_data(
failure_score: float = 0,
format_failure_score: float = -1,
log_format_failures: bool = False,
+ callback_metadata: dict[str, Any] | None = None,
) -> list[TraceData]:
# Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score
# (if metric != None)
@@ -110,7 +111,11 @@ def bootstrap_trace_data(
program.forward = MethodType(patched_forward, program)
try:
- results = evaluator(program, metric=wrapped_metric).results
+ results = evaluator(
+ program,
+ metric=wrapped_metric,
+ callback_metadata=callback_metadata,
+ ).results
finally:
program.forward = original_forward
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.py
index 50f8b6ea7..0e483034e 100644
--- a/dspy/teleprompt/gepa/gepa_utils.py
+++ b/dspy/teleprompt/gepa/gepa_utils.py
@@ -119,8 +119,10 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
if capture_traces:
# bootstrap_trace_data-like flow with trace capture
- from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
- trajs = bootstrap_trace_data(
+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module
+
+ eval_callback_metadata = {"disable_logging": True}
+ trajs = bootstrap_trace_module.bootstrap_trace_data(
program=program,
dataset=batch,
metric=self.metric_fn,
@@ -129,6 +131,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
capture_failed_parses=True,
failure_score=self.failure_score,
format_failure_score=self.failure_score,
+ callback_metadata=eval_callback_metadata,
)
scores = []
outputs = []
diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.py
index 3a1e3d10e..020f4cba3 100644
--- a/tests/teleprompt/test_bootstrap_trace.py
+++ b/tests/teleprompt/test_bootstrap_trace.py
@@ -1,3 +1,4 @@
+from typing import Any
from unittest import mock
from litellm import Choices, Message, ModelResponse
@@ -118,3 +119,35 @@ def test_bootstrap_trace_data():
# Each trace entry should be a tuple of (predictor, inputs, prediction)
for trace_entry in result["trace"]:
assert len(trace_entry) == 3, "Trace entry should have 3 elements"
+
+
+def test_bootstrap_trace_data_passes_callback_metadata(monkeypatch):
+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module
+
+ class DummyProgram(dspy.Module):
+ def forward(self, **kwargs): # pragma: no cover - stub forward
+ return dspy.Prediction()
+
+ captured_metadata: dict[str, Any] = {}
+
+ class DummyEvaluate:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __call__(self, *args, callback_metadata=None, **kwargs):
+ captured_metadata["value"] = callback_metadata
+
+ class _Result:
+ results: list[Any] = []
+
+ return _Result()
+
+ monkeypatch.setattr(bootstrap_trace_module, "Evaluate", DummyEvaluate)
+
+ bootstrap_trace_module.bootstrap_trace_data(
+ program=DummyProgram(),
+ dataset=[],
+ callback_metadata={"disable_logging": True},
+ )
+
+ assert captured_metadata["value"] == {"disable_logging": True}
diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.py
index b78e80894..2afe31bb9 100644
--- a/tests/teleprompt/test_gepa.py
+++ b/tests/teleprompt/test_gepa.py
@@ -43,6 +43,40 @@ def bad_metric(example, prediction):
return 0.0
+def test_gepa_adapter_disables_logging_during_trace_capture(monkeypatch):
+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module
+ from dspy.teleprompt.gepa import gepa_utils
+
+ class DummyModule(dspy.Module):
+ def forward(self, **kwargs): # pragma: no cover - stub forward
+ return dspy.Prediction()
+
+ # Exercise the adapter evaluate path directly.
+ adapter = gepa_utils.DspyAdapter(
+ student_module=SimpleModule("input -> output"),
+ metric_fn=simple_metric,
+ feedback_map={},
+ failure_score=0.0,
+ )
+
+ captured_kwargs: dict[str, Any] = {}
+
+ def dummy_bootstrap_trace_data(*args, **kwargs):
+ captured_kwargs.update(kwargs)
+ return []
+
+ monkeypatch.setattr(bootstrap_trace_module, "bootstrap_trace_data", dummy_bootstrap_trace_data)
+ monkeypatch.setattr(
+ gepa_utils.DspyAdapter,
+ "build_program",
+ lambda self, candidate: DummyModule(),
+ )
+
+ adapter.evaluate(batch=[], candidate={}, capture_traces=True)
+
+ assert captured_kwargs["callback_metadata"] == {"disable_logging": True}
+
+
@pytest.fixture
def mock_mlflow():
mock_mlflow = mock.MagicMock()