Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

[Fork of #8665] Fix mlflow gepa (#8691)

Chen Qian84a4b7e598f8
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.py
index 5cc3b544a..5f1788099 100644
--- a/dspy/teleprompt/__init__.py
+++ b/dspy/teleprompt/__init__.py
@@ -2,6 +2,7 @@ from dspy.teleprompt.avatar_optimizer import AvatarOptimizer
from dspy.teleprompt.bettertogether import BetterTogether
from dspy.teleprompt.bootstrap import BootstrapFewShot
from dspy.teleprompt.bootstrap_finetune import BootstrapFinetune
+from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
from dspy.teleprompt.copro_optimizer import COPRO
from dspy.teleprompt.ensemble import Ensemble
from dspy.teleprompt.infer_rules import InferRules
@@ -30,4 +31,5 @@ __all__ = [
"LabeledFewShot",
"InferRules",
"SIMBA",
+ "bootstrap_trace_data",
]
diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.py
index 5f0778638..6a431e355 100644
--- a/dspy/teleprompt/bootstrap_finetune.py
+++ b/dspy/teleprompt/bootstrap_finetune.py
@@ -1,7 +1,6 @@
import logging
from collections import defaultdict
-from dataclasses import dataclass
-from typing import Any, Callable, TypedDict
+from typing import Any, Callable
import dspy
from dspy.adapters.base import Adapter
@@ -9,13 +8,11 @@ from dspy.adapters.chat_adapter import ChatAdapter
from dspy.clients.lm import LM
from dspy.clients.utils_finetune import infer_data_format
from dspy.dsp.utils.settings import settings
-from dspy.evaluate.evaluate import Evaluate
from dspy.predict.predict import Predict
from dspy.primitives.example import Example
from dspy.primitives.module import Module
-from dspy.primitives.prediction import Prediction
+from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
from dspy.teleprompt.teleprompt import Teleprompter
-from dspy.utils.exceptions import AdapterParseError
logger = logging.getLogger(__name__)
@@ -216,119 +213,6 @@ def build_call_data_from_trace(
return call_data
-@dataclass
-class FailedPrediction:
- completion_text: str
- format_reward: float | None = None
-
-class TraceData(TypedDict):
- example_ind: int
- example: Example
- prediction: Prediction
- trace: list[tuple[Any, dict[str, Any], Prediction]]
- score: float | None
-
-def bootstrap_trace_data(
- program: Module,
- dataset: list[Example],
- metric: Callable | None = None,
- num_threads: int | None = None,
- raise_on_error=True,
- capture_failed_parses=False,
- failure_score: float = 0,
- format_failure_score: float = -1,
- log_format_failures: bool = False,
-) -> list[TraceData]:
- # Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score
- # (if metric != None)
- evaluator = Evaluate(
- devset=dataset,
- num_threads=num_threads,
- display_progress=True,
- provide_traceback=False, # TODO(check with team)
- max_errors=len(dataset) * 10, # TODO(check with team)
- failure_score=failure_score,
- )
-
- def wrapped_metric(example, prediction, trace=None):
- prediction, _ = prediction
- if isinstance(prediction, FailedPrediction):
- return prediction.format_reward or format_failure_score
- return metric(example, prediction, trace) if metric else True
-
- def wrapped_program(**kwargs):
- with dspy.context(trace=[]):
- try:
- return program(**kwargs), dspy.settings.trace.copy()
- except AdapterParseError as e:
- completion_str = e.lm_response
- parsed_result = e.parsed_result
- failed_signature = e.signature
- failed_inputs = kwargs
-
- present = list(parsed_result.keys()) if parsed_result else None
- expected = list(failed_signature.output_fields.keys())
-
- found_pred = None
- for pred in program.predictors():
- if pred.signature == failed_signature:
- found_pred = pred
- break
- if found_pred is None:
- raise ValueError(f"Failed to find the predictor for the failed signature: {failed_signature}")
-
- trace = dspy.settings.trace.copy()
- # Trace is Tuple[signature, inputs, prediction outputs]
- if present:
- failed_pred = FailedPrediction(
- completion_text=completion_str,
- format_reward=format_failure_score
- + (failure_score - format_failure_score) * (present / expected),
- )
- else:
- failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)
-
- trace.append(
- (
- found_pred,
- failed_inputs,
- failed_pred,
- )
- )
-
- if log_format_failures:
- logging.warning(
- "Failed to parse output for example. This is likely due to the LLM response not following the adapter's formatting."
- )
-
- return failed_pred, trace
-
- results = evaluator(wrapped_program, metric=wrapped_metric).results
-
- data = []
- for example_ind, (example, prediction, score) in enumerate(results):
- try:
- prediction, trace = prediction
- except ValueError as ve:
- # TODO(GRPO Team): Often during GRPO bootstrapping, the LLM response does not follow dspy formatting. This leads to a value error.
- # To reproduce this issue, try Qwen/Qwen2.5-Coder-0.5B-Instruct with MATH dataset
- # Proposal(Lakshya): We should capture the incorrectly-formatted LLM response, and store it in the trace, and pass it to in the GRPO group
- # with a high-negative user-configurable score.
- logger.warning(
- "Failed to unpack prediction and trace. This is likely due to the LLM response not following dspy formatting."
- )
- if raise_on_error:
- raise ve
- else:
- continue
- data_dict = {"example": example, "prediction": prediction, "trace": trace, "example_ind": example_ind}
- if metric:
- data_dict["score"] = score
- data.append(data_dict)
-
- return data
-
-
# # TODO(PR) check with team
# def bootstrap_trace_data_one_example(
# example: Example,
diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.py
new file mode 100644
index 000000000..d3c81db4a
--- /dev/null
+++ b/dspy/teleprompt/bootstrap_trace.py
@@ -0,0 +1,134 @@
+import logging
+from dataclasses import dataclass
+from types import MethodType
+from typing import Any, Callable, TypedDict
+
+import dspy
+from dspy.evaluate.evaluate import Evaluate
+from dspy.primitives.example import Example
+from dspy.primitives.module import Module
+from dspy.primitives.prediction import Prediction
+from dspy.utils.exceptions import AdapterParseError
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FailedPrediction:
+ completion_text: str
+ format_reward: float | None = None
+
+
+class TraceData(TypedDict):
+ example_ind: int
+ example: Example
+ prediction: Prediction
+ trace: list[tuple[Any, dict[str, Any], Prediction]]
+ score: float | None
+
+
+def bootstrap_trace_data(
+ program: Module,
+ dataset: list[Example],
+ metric: Callable | None = None,
+ num_threads: int | None = None,
+ raise_on_error=True,
+ capture_failed_parses=False,
+ failure_score: float = 0,
+ format_failure_score: float = -1,
+ log_format_failures: bool = False,
+) -> list[TraceData]:
+ # Return a list of dicts with the following keys: example_ind, example, prediction, trace, and score
+ # (if metric != None)
+ evaluator = Evaluate(
+ devset=dataset,
+ num_threads=num_threads,
+ display_progress=True,
+ provide_traceback=False, # TODO(check with team)
+ max_errors=len(dataset) * 10, # TODO(check with team)
+ failure_score=failure_score,
+ )
+
+ def wrapped_metric(example, prediction, trace=None):
+ prediction, _ = prediction
+ if isinstance(prediction, FailedPrediction):
+ return prediction.format_reward or format_failure_score
+ return metric(example, prediction, trace) if metric else True
+
+ original_forward = program.forward
+
+ def patched_forward(program_to_use: Module, **kwargs):
+ with dspy.context(trace=[]):
+ try:
+ return original_forward(**kwargs), dspy.settings.trace.copy()
+ except AdapterParseError as e:
+ completion_str = e.lm_response
+ parsed_result = e.parsed_result
+ failed_signature = e.signature
+ failed_inputs = kwargs
+
+ present = list(parsed_result.keys()) if parsed_result else None
+ expected = list(failed_signature.output_fields.keys())
+
+ found_pred = None
+ for pred in program_to_use.predictors():
+ if pred.signature == failed_signature:
+ found_pred = pred
+ break
+ if found_pred is None:
+ raise ValueError(f"Failed to find the predictor for the failed signature: {failed_signature}")
+
+ trace = dspy.settings.trace.copy()
+ # Trace is Tuple[signature, inputs, prediction outputs]
+ if present:
+ failed_pred = FailedPrediction(
+ completion_text=completion_str,
+ format_reward=format_failure_score
+ + (failure_score - format_failure_score) * (present / expected),
+ )
+ else:
+ failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)
+
+ trace.append(
+ (
+ found_pred,
+ failed_inputs,
+ failed_pred,
+ )
+ )
+
+ if log_format_failures:
+ logging.warning(
+ "Failed to parse output for example. This is likely due to the LLM response not following "
+ "the adapter's formatting."
+ )
+
+ return failed_pred, trace
+
+ program.forward = MethodType(patched_forward, program)
+
+ results = evaluator(program, metric=wrapped_metric).results
+
+ data = []
+ for example_ind, (example, prediction, score) in enumerate(results):
+ try:
+ prediction, trace = prediction
+ except ValueError as ve:
+ # TODO(GRPO Team): Often during GRPO bootstrapping, the LLM response does not follow dspy formatting. This
+ # leads to a value error. To reproduce this issue, try Qwen/Qwen2.5-Coder-0.5B-Instruct with MATH dataset.
+ # Proposal(Lakshya): We should capture the incorrectly-formatted LLM response, and store it in the trace,
+ # and pass it to in the GRPO group with a high-negative user-configurable score.
+ logger.warning(
+ "Failed to unpack prediction and trace. This is likely due to the LLM response not following "
+ "dspy formatting."
+ )
+ if raise_on_error:
+ raise ve
+ else:
+ continue
+ data_dict = {"example": example, "prediction": prediction, "trace": trace, "example_ind": example_ind}
+ if metric:
+ data_dict["score"] = score
+ data.append(data_dict)
+
+ return data
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.py
index f4802dfdf..95d1fc17f 100644
--- a/dspy/teleprompt/gepa/gepa_utils.py
+++ b/dspy/teleprompt/gepa/gepa_utils.py
@@ -8,7 +8,7 @@ from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.types import History
from dspy.evaluate import Evaluate
from dspy.primitives import Example, Prediction
-from dspy.teleprompt.bootstrap_finetune import TraceData
+from dspy.teleprompt.bootstrap_trace import TraceData
class LoggerAdapter:
@@ -82,7 +82,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
if capture_traces:
# bootstrap_trace_data-like flow with trace capture
- from ..bootstrap_finetune import bootstrap_trace_data
+ from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
trajs = bootstrap_trace_data(
program=program,
dataset=batch,
@@ -122,7 +122,7 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
return EvaluationBatch(outputs=outputs, scores=scores, trajectories=None)
def make_reflective_dataset(self, candidate, eval_batch, components_to_update):
- from ..bootstrap_finetune import FailedPrediction
+ from dspy.teleprompt.bootstrap_trace import FailedPrediction
program = self.build_program(candidate)
ret_d: dict[str, list[dict[str, Any]]] = {}
diff --git a/dspy/teleprompt/grpo.py b/dspy/teleprompt/grpo.py
index 3f929f4cd..9b58aac3f 100644
--- a/dspy/teleprompt/grpo.py
+++ b/dspy/teleprompt/grpo.py
@@ -12,12 +12,11 @@ from dspy.evaluate.evaluate import Evaluate
from dspy.primitives.example import Example
from dspy.primitives.module import Module
from dspy.teleprompt.bootstrap_finetune import (
- FailedPrediction,
FinetuneTeleprompter,
all_predictors_have_lms,
assert_structural_equivalency,
- bootstrap_trace_data,
)
+from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data
logger = logging.getLogger(__name__)
diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.py
new file mode 100644
index 000000000..3a1e3d10e
--- /dev/null
+++ b/tests/teleprompt/test_bootstrap_trace.py
@@ -0,0 +1,120 @@
+from unittest import mock
+
+from litellm import Choices, Message, ModelResponse
+
+import dspy
+from dspy.primitives.example import Example
+from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data
+
+
+def test_bootstrap_trace_data():
+ """Test bootstrap_trace_data function with single dspy.Predict program."""
+
+ # Define signature for string -> int conversion
+ class StringToIntSignature(dspy.Signature):
+ """Convert a string number to integer"""
+
+ text: str = dspy.InputField()
+ number: int = dspy.OutputField()
+
+ # Create program with single dspy.Predict
+ program = dspy.Predict(StringToIntSignature)
+
+ # Create dummy dataset of size 5
+ dataset = [
+ Example(text="one", number=1).with_inputs("text"),
+ Example(text="two", number=2).with_inputs("text"),
+ Example(text="three", number=3).with_inputs("text"),
+ Example(text="four", number=4).with_inputs("text"),
+ Example(text="five", number=5).with_inputs("text"),
+ ]
+
+ # Define exact match metric
+ def exact_match_metric(example, prediction, trace=None):
+ return example.number == prediction.number
+
+ # Configure dspy
+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
+
+ # Mock litellm completion responses
+ # 4 successful responses and 1 that will trigger AdapterParseError
+ successful_responses = [
+ ModelResponse(
+ choices=[Choices(message=Message(content='```json\n{"number": 1}\n```'))],
+ model="openai/gpt-4o-mini",
+ ),
+ ModelResponse(
+ choices=[Choices(message=Message(content='```json\n{"number": 2}\n```'))],
+ model="openai/gpt-4o-mini",
+ ),
+ ModelResponse(
+ choices=[Choices(message=Message(content='```json\n{"number": 3}\n```'))],
+ model="openai/gpt-4o-mini",
+ ),
+ ModelResponse(
+ choices=[Choices(message=Message(content='```json\n{"number": 4}\n```'))],
+ model="openai/gpt-4o-mini",
+ ),
+ ]
+
+ # Create a side effect that will trigger AdapterParseError on the 3rd call (index 2)
+ def completion_side_effect(*args, **kwargs):
+ call_count = completion_side_effect.call_count
+ completion_side_effect.call_count += 1
+
+ if call_count == 5: # Third call (0-indexed)
+ # Return malformed response that will cause AdapterParseError
+ return ModelResponse(
+ choices=[Choices(message=Message(content="This is an invalid JSON!"))],
+ model="openai/gpt-4o-mini",
+ )
+ else:
+ return successful_responses[call_count]
+
+ completion_side_effect.call_count = 0
+
+ with mock.patch("litellm.completion", side_effect=completion_side_effect):
+ # Call bootstrap_trace_data
+ results = bootstrap_trace_data(
+ program=program,
+ dataset=dataset,
+ metric=exact_match_metric,
+ raise_on_error=False,
+ capture_failed_parses=True,
+ )
+
+ # Verify results
+ assert len(results) == 5, f"Expected 5 results, got {len(results)}"
+
+ # Count successful and failed predictions
+ successful_count = 0
+ failed_count = 0
+
+ for result in results:
+ assert "example" in result
+ assert "prediction" in result
+ assert "trace" in result
+ assert "example_ind" in result
+ assert "score" in result
+
+ if isinstance(result["prediction"], FailedPrediction):
+ failed_count += 1
+ # Verify failed prediction structure
+ assert hasattr(result["prediction"], "completion_text")
+ assert hasattr(result["prediction"], "format_reward")
+ assert result["prediction"].completion_text == "This is an invalid JSON!"
+ else:
+ successful_count += 1
+ # Verify successful prediction structure
+ assert hasattr(result["prediction"], "number")
+
+ # Verify we have the expected number of successful and failed bootstrapping
+ assert successful_count == 4, f"Expected 4 successful predictions, got {successful_count}"
+ assert failed_count == 1, f"Expected 1 failed prediction, got {failed_count}"
+
+ # Verify that traces are present
+ for result in results:
+ assert len(result["trace"]) > 0, "Trace should not be empty"
+ # Each trace entry should be a tuple of (predictor, inputs, prediction)
+ for trace_entry in result["trace"]:
+ assert len(trace_entry) == 3, "Trace entry should have 3 elements"