DSPy: The framework for programming—not prompting—language models
Python
[Fork of #8665] Fix mlflow gepa (#8691)
Chen Qian84a4b7e598f8
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.pyindex 5cc3b544a..5f1788099 100644--- a/dspy/teleprompt/__init__.py+++ b/dspy/teleprompt/__init__.py@@ -2,6 +2,7 @@ from dspy.teleprompt.avatar_optimizer import AvatarOptimizerfrom dspy.teleprompt.bettertogether import BetterTogetherfrom dspy.teleprompt.bootstrap import BootstrapFewShotfrom dspy.teleprompt.bootstrap_finetune import BootstrapFinetune+from dspy.teleprompt.bootstrap_trace import bootstrap_trace_datafrom dspy.teleprompt.copro_optimizer import COPROfrom dspy.teleprompt.ensemble import Ensemblefrom dspy.teleprompt.infer_rules import InferRules@@ -30,4 +31,5 @@ __all__ = ["LabeledFewShot","InferRules","SIMBA",+ "bootstrap_trace_data",]diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.pyindex 5f0778638..6a431e355 100644--- a/dspy/teleprompt/bootstrap_finetune.py+++ b/dspy/teleprompt/bootstrap_finetune.py@@ -1,7 +1,6 @@import loggingfrom collections import defaultdict-from dataclasses import dataclass-from typing import Any, Callable, TypedDict+from typing import Any, Callableimport dspyfrom dspy.adapters.base import Adapter@@ -9,13 +8,11 @@ from dspy.adapters.chat_adapter import ChatAdapterfrom dspy.clients.lm import LMfrom dspy.clients.utils_finetune import infer_data_formatfrom dspy.dsp.utils.settings import settings-from dspy.evaluate.evaluate import Evaluatefrom dspy.predict.predict import Predictfrom dspy.primitives.example import Examplefrom dspy.primitives.module import Module-from dspy.primitives.prediction import Prediction+from dspy.teleprompt.bootstrap_trace import bootstrap_trace_datafrom dspy.teleprompt.teleprompt import Teleprompter-from dspy.utils.exceptions import AdapterParseErrorlogger = logging.getLogger(__name__)@@ -216,119 +213,6 @@ def build_call_data_from_trace(return call_data-@dataclass-class FailedPrediction:- completion_text: str- format_reward: float | None = None--class TraceData(TypedDict):- example_ind: int- example: Example- prediction: Prediction- trace: list[tuple[Any, dict[str, Any], Prediction]]- score: float | None--def bootstrap_trace_data(- program: Module,- dataset: list[Example],- metric: Callable | None = None,- num_threads: int | None = None,- raise_on_error=True,- capture_failed_parses=False,- failure_score: float = 0,- format_failure_score: float = -1,- log_format_failures: bool = False,-) -> list[TraceData]:- # Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score- # (if metric != None)- evaluator = Evaluate(- devset=dataset,- num_threads=num_threads,- display_progress=True,- provide_traceback=False, # TODO(check with team)- max_errors=len(dataset) * 10, # TODO(check with team)- failure_score=failure_score,- )-- def wrapped_metric(example, prediction, trace=None):- prediction, _ = prediction- if isinstance(prediction, FailedPrediction):- return prediction.format_reward or format_failure_score- return metric(example, prediction, trace) if metric else True-- def wrapped_program(**kwargs):- with dspy.context(trace=[]):- try:- return program(**kwargs), dspy.settings.trace.copy()- except AdapterParseError as e:- completion_str = e.lm_response- parsed_result = e.parsed_result- failed_signature = e.signature- failed_inputs = kwargs-- present = list(parsed_result.keys()) if parsed_result else None- expected = list(failed_signature.output_fields.keys())-- found_pred = None- for pred in program.predictors():- if pred.signature == failed_signature:- found_pred = pred- break- if found_pred is None:- raise ValueError(f"Failed to find the predictor for the failed signature: {failed_signature}")-- trace = dspy.settings.trace.copy()- # Trace is Tuple[signature, inputs, prediction outputs]- if present:- failed_pred = FailedPrediction(- completion_text=completion_str,- format_reward=format_failure_score- + (failure_score - format_failure_score) * (present / expected),- )- else:- failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)-- trace.append(- (- found_pred,- failed_inputs,- failed_pred,- )- )-- if log_format_failures:- logging.warning(- "Failed to parse output for example. This is likely due to the LLM response not following the adapter's formatting."- )-- return failed_pred, trace-- results = evaluator(wrapped_program, metric=wrapped_metric).results-- data = []- for example_ind, (example, prediction, score) in enumerate(results):- try:- prediction, trace = prediction- except ValueError as ve:- # TODO(GRPO Team): Often during GRPO bootstrapping, the LLM response does not follow dspy formatting. This leads to a value error.- # To reproduce this issue, try Qwen/Qwen2.5-Coder-0.5B-Instruct with MATH dataset- # Proposal(Lakshya): We should capture the incorrectly-formatted LLM response, and store it in the trace, and pass it to in the GRPO group- # with a high-negative user-configurable score.- logger.warning(- "Failed to unpack prediction and trace. This is likely due to the LLM response not following dspy formatting."- )- if raise_on_error:- raise ve- else:- continue- data_dict = {"example": example, "prediction": prediction, "trace": trace, "example_ind": example_ind}- if metric:- data_dict["score"] = score- data.append(data_dict)-- return data--# # TODO(PR) check with team# def bootstrap_trace_data_one_example(# example: Example,diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.pynew file mode 100644index 000000000..d3c81db4a--- /dev/null+++ b/dspy/teleprompt/bootstrap_trace.py@@ -0,0 +1,134 @@+import logging+from dataclasses import dataclass+from types import MethodType+from typing import Any, Callable, TypedDict++import dspy+from dspy.evaluate.evaluate import Evaluate+from dspy.primitives.example import Example+from dspy.primitives.module import Module+from dspy.primitives.prediction import Prediction+from dspy.utils.exceptions import AdapterParseError++logger = logging.getLogger(__name__)+++@dataclass+class FailedPrediction:+ completion_text: str+ format_reward: float | None = None+++class TraceData(TypedDict):+ example_ind: int+ example: Example+ prediction: Prediction+ trace: list[tuple[Any, dict[str, Any], Prediction]]+ score: float | None+++def bootstrap_trace_data(+ program: Module,+ dataset: list[Example],+ metric: Callable | None = None,+ num_threads: int | None = None,+ raise_on_error=True,+ capture_failed_parses=False,+ failure_score: float = 0,+ format_failure_score: float = -1,+ log_format_failures: bool = False,+) -> list[TraceData]:+ # Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score+ # (if metric != None)+ evaluator = Evaluate(+ devset=dataset,+ num_threads=num_threads,+ display_progress=True,+ provide_traceback=False, # TODO(check with team)+ max_errors=len(dataset) * 10, # TODO(check with team)+ failure_score=failure_score,+ )++ def wrapped_metric(example, prediction, trace=None):+ prediction, _ = prediction+ if isinstance(prediction, FailedPrediction):+ return prediction.format_reward or format_failure_score+ return metric(example, prediction, trace) if metric else True++ original_forward = program.forward++ def patched_forward(program_to_use: Module, **kwargs):+ with dspy.context(trace=[]):+ try:+ return original_forward(**kwargs), dspy.settings.trace.copy()+ except AdapterParseError as e:+ completion_str = e.lm_response+ parsed_result = e.parsed_result+ failed_signature = e.signature+ failed_inputs = kwargs++ present = list(parsed_result.keys()) if parsed_result else None+ expected = list(failed_signature.output_fields.keys())++ found_pred = None+ for pred in program_to_use.predictors():+ if pred.signature == failed_signature:+ found_pred = pred+ break+ if found_pred is None:+ raise ValueError(f"Failed to find the predictor for the failed signature: {failed_signature}")++ trace = dspy.settings.trace.copy()+ # Trace is Tuple[signature, inputs, prediction outputs]+ if present:+ failed_pred = FailedPrediction(+ completion_text=completion_str,+ format_reward=format_failure_score+ + (failure_score - format_failure_score) * (present / expected),+ )+ else:+ failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)++ trace.append(+ (+ found_pred,+ failed_inputs,+ failed_pred,+ )+ )++ if log_format_failures:+ logging.warning(+ "Failed to parse output for example. This is likely due to the LLM response not following "+ "the adapter's formatting."+ )++ return failed_pred, trace++ program.forward = MethodType(patched_forward, program)++ results = evaluator(program, metric=wrapped_metric).results++ data = []+ for example_ind, (example, prediction, score) in enumerate(results):+ try:+ prediction, trace = prediction+ except ValueError as ve:+ # TODO(GRPO Team): Often during GRPO bootstrapping, the LLM response does not follow dspy formatting. This+ # leads to a value error. To reproduce this issue, try Qwen/Qwen2.5-Coder-0.5B-Instruct with MATH dataset.+ # Proposal(Lakshya): We should capture the incorrectly-formatted LLM response, and store it in the trace,+ # and pass it to in the GRPO group with a high-negative user-configurable score.+ logger.warning(+ "Failed to unpack prediction and trace. This is likely due to the LLM response not following "+ "dspy formatting."+ )+ if raise_on_error:+ raise ve+ else:+ continue+ data_dict = {"example": example, "prediction": prediction, "trace": trace, "example_ind": example_ind}+ if metric:+ data_dict["score"] = score+ data.append(data_dict)++ return datadiff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.pyindex f4802dfdf..95d1fc17f 100644--- a/dspy/teleprompt/gepa/gepa_utils.py+++ b/dspy/teleprompt/gepa/gepa_utils.py@@ -8,7 +8,7 @@ from dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.types import Historyfrom dspy.evaluate import Evaluatefrom dspy.primitives import Example, Prediction-from dspy.teleprompt.bootstrap_finetune import TraceData+from dspy.teleprompt.bootstrap_trace import TraceDataclass LoggerAdapter:@@ -82,7 +82,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):if capture_traces:# bootstrap_trace_data-like flow with trace capture- from ..bootstrap_finetune import bootstrap_trace_data+ from dspy.teleprompt.bootstrap_trace import bootstrap_trace_datatrajs = bootstrap_trace_data(program=program,dataset=batch,@@ -122,7 +122,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):return EvaluationBatch(outputs=outputs, scores=scores, trajectories=None)def make_reflective_dataset(self, candidate, eval_batch, components_to_update):- from ..bootstrap_finetune import FailedPrediction+ from dspy.teleprompt.bootstrap_trace import FailedPredictionprogram = self.build_program(candidate)ret_d: dict[str, list[dict[str, Any]]] = {}diff --git a/dspy/teleprompt/grpo.py b/dspy/teleprompt/grpo.pyindex 3f929f4cd..9b58aac3f 100644--- a/dspy/teleprompt/grpo.py+++ b/dspy/teleprompt/grpo.py@@ -12,12 +12,11 @@ from dspy.evaluate.evaluate import Evaluatefrom dspy.primitives.example import Examplefrom dspy.primitives.module import Modulefrom dspy.teleprompt.bootstrap_finetune import (- FailedPrediction,FinetuneTeleprompter,all_predictors_have_lms,assert_structural_equivalency,- bootstrap_trace_data,)+from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_datalogger = logging.getLogger(__name__)diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.pynew file mode 100644index 000000000..3a1e3d10e--- /dev/null+++ b/tests/teleprompt/test_bootstrap_trace.py@@ -0,0 +1,120 @@+from unittest import mock++from litellm import Choices, Message, ModelResponse++import dspy+from dspy.primitives.example import Example+from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data+++def test_bootstrap_trace_data():+ """Test bootstrap_trace_data function with single dspy.Predict program."""++ # Define signature for string -> int conversion+ class StringToIntSignature(dspy.Signature):+ """Convert a string number to integer"""++ text: str = dspy.InputField()+ number: int = dspy.OutputField()++ # Create program with single dspy.Predict+ program = dspy.Predict(StringToIntSignature)++ # Create dummy dataset of size 5+ dataset = [+ Example(text="one", number=1).with_inputs("text"),+ Example(text="two", number=2).with_inputs("text"),+ Example(text="three", number=3).with_inputs("text"),+ Example(text="four", number=4).with_inputs("text"),+ Example(text="five", number=5).with_inputs("text"),+ ]++ # Define exact match metric+ def exact_match_metric(example, prediction, trace=None):+ return example.number == prediction.number++ # Configure dspy+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())++ # Mock litellm completion responses+ # 4 successful responses and 1 that will trigger AdapterParseError+ successful_responses = [+ ModelResponse(+ choices=[Choices(message=Message(content='```json\n{"number": 1}\n```'))],+ model="openai/gpt-4o-mini",+ ),+ ModelResponse(+ choices=[Choices(message=Message(content='```json\n{"number": 2}\n```'))],+ model="openai/gpt-4o-mini",+ ),+ ModelResponse(+ choices=[Choices(message=Message(content='```json\n{"number": 3}\n```'))],+ model="openai/gpt-4o-mini",+ ),+ ModelResponse(+ choices=[Choices(message=Message(content='```json\n{"number": 4}\n```'))],+ model="openai/gpt-4o-mini",+ ),+ ]++ # Create a side effect that will trigger AdapterParseError on the 3rd call (index 2)+ def completion_side_effect(*args, **kwargs):+ call_count = completion_side_effect.call_count+ completion_side_effect.call_count += 1++ if call_count == 5: # Third call (0-indexed)+ # Return malformed response that will cause AdapterParseError+ return ModelResponse(+ choices=[Choices(message=Message(content="This is an invalid JSON!"))],+ model="openai/gpt-4o-mini",+ )+ else:+ return successful_responses[call_count]++ completion_side_effect.call_count = 0++ with mock.patch("litellm.completion", side_effect=completion_side_effect):+ # Call bootstrap_trace_data+ results = bootstrap_trace_data(+ program=program,+ dataset=dataset,+ metric=exact_match_metric,+ raise_on_error=False,+ capture_failed_parses=True,+ )++ # Verify results+ assert len(results) == 5, f"Expected 5 results, got {len(results)}"++ # Count successful and failed predictions+ successful_count = 0+ failed_count = 0++ for result in results:+ assert "example" in result+ assert "prediction" in result+ assert "trace" in result+ assert "example_ind" in result+ assert "score" in result++ if isinstance(result["prediction"], FailedPrediction):+ failed_count += 1+ # Verify failed prediction structure+ assert hasattr(result["prediction"], "completion_text")+ assert hasattr(result["prediction"], "format_reward")+ assert result["prediction"].completion_text == "This is an invalid JSON!"+ else:+ successful_count += 1+ # Verify successful prediction structure+ assert hasattr(result["prediction"], "number")++ # Verify we have the expected number of successful and failed bootstrapping+ assert successful_count == 4, f"Expected 4 successful predictions, got {successful_count}"+ assert failed_count == 1, f"Expected 1 failed prediction, got {failed_count}"++ # Verify that traces are present+ for result in results:+ assert len(result["trace"]) > 0, "Trace should not be empty"+ # Each trace entry should be a tuple of (predictor, inputs, prediction)+ for trace_entry in result["trace"]:+ assert len(trace_entry) == 3, "Trace entry should have 3 elements"