DSPy: The framework for programming—not prompting—language models
Python
Limit callback metadata to trace capture path (#8835)
TomuHirata71fc252e1d0b
diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.pyindex 5af24223c..2f0cc60cd 100644--- a/dspy/teleprompt/bootstrap_trace.py+++ b/dspy/teleprompt/bootstrap_trace.py@@ -37,6 +37,7 @@ def bootstrap_trace_data(failure_score: float = 0,format_failure_score: float = -1,log_format_failures: bool = False,+ callback_metadata: dict[str, Any] | None = None,) -> list[TraceData]:# Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score# (if metric != None)@@ -110,7 +111,11 @@ def bootstrap_trace_data(program.forward = MethodType(patched_forward, program)try:- results = evaluator(program, metric=wrapped_metric).results+ results = evaluator(+ program,+ metric=wrapped_metric,+ callback_metadata=callback_metadata,+ ).resultsfinally:program.forward = original_forwarddiff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.pyindex 50f8b6ea7..0e483034e 100644--- a/dspy/teleprompt/gepa/gepa_utils.py+++ b/dspy/teleprompt/gepa/gepa_utils.py@@ -119,8 +119,10 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):if capture_traces:# bootstrap_trace_data-like flow with trace capture- from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data- trajs = bootstrap_trace_data(+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module++ eval_callback_metadata = {"disable_logging": True}+ trajs = bootstrap_trace_module.bootstrap_trace_data(program=program,dataset=batch,metric=self.metric_fn,@@ -129,6 +131,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):capture_failed_parses=True,failure_score=self.failure_score,format_failure_score=self.failure_score,+ callback_metadata=eval_callback_metadata,)scores = []outputs = []diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.pyindex 3a1e3d10e..020f4cba3 100644--- a/tests/teleprompt/test_bootstrap_trace.py+++ b/tests/teleprompt/test_bootstrap_trace.py@@ -1,3 +1,4 @@+from typing import Anyfrom unittest import mockfrom litellm import Choices, Message, ModelResponse@@ -118,3 +119,35 @@ def test_bootstrap_trace_data():# Each trace entry should be a tuple of (predictor, inputs, prediction)for trace_entry in result["trace"]:assert len(trace_entry) == 3, "Trace entry should have 3 elements"+++def test_bootstrap_trace_data_passes_callback_metadata(monkeypatch):+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module++ class DummyProgram(dspy.Module):+ def forward(self, **kwargs): # pragma: no cover - stub forward+ return dspy.Prediction()++ captured_metadata: dict[str, Any] = {}++ class DummyEvaluate:+ def __init__(self, *args, **kwargs):+ pass++ def __call__(self, *args, callback_metadata=None, **kwargs):+ captured_metadata["value"] = callback_metadata++ class _Result:+ results: list[Any] = []++ return _Result()++ monkeypatch.setattr(bootstrap_trace_module, "Evaluate", DummyEvaluate)++ bootstrap_trace_module.bootstrap_trace_data(+ program=DummyProgram(),+ dataset=[],+ callback_metadata={"disable_logging": True},+ )++ assert captured_metadata["value"] == {"disable_logging": True}diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.pyindex b78e80894..2afe31bb9 100644--- a/tests/teleprompt/test_gepa.py+++ b/tests/teleprompt/test_gepa.py@@ -43,6 +43,40 @@ def bad_metric(example, prediction):return 0.0+def test_gepa_adapter_disables_logging_during_trace_capture(monkeypatch):+ from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module+ from dspy.teleprompt.gepa import gepa_utils++ class DummyModule(dspy.Module):+ def forward(self, **kwargs): # pragma: no cover - stub forward+ return dspy.Prediction()++ # Exercise the adapter evaluate path directly.+ adapter = gepa_utils.DspyAdapter(+ student_module=SimpleModule("input -> output"),+ metric_fn=simple_metric,+ feedback_map={},+ failure_score=0.0,+ )++ captured_kwargs: dict[str, Any] = {}++ def dummy_bootstrap_trace_data(*args, **kwargs):+ captured_kwargs.update(kwargs)+ return []++ monkeypatch.setattr(bootstrap_trace_module, "bootstrap_trace_data", dummy_bootstrap_trace_data)+ monkeypatch.setattr(+ gepa_utils.DspyAdapter,+ "build_program",+ lambda self, candidate: DummyModule(),+ )++ adapter.evaluate(batch=[], candidate={}, capture_traces=True)++ assert captured_kwargs["callback_metadata"] == {"disable_logging": True}++@pytest.fixturedef mock_mlflow():mock_mlflow = mock.MagicMock()