Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
7.2 KB
Raw Blame
from typing import Any, ClassVar
from unittest import mock

import pytest
from litellm import Choices, Message, ModelResponse

import dspy
from dspy.primitives.example import Example
from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data
from dspy.utils import DummyLM


def test_bootstrap_trace_data():
    """Test bootstrap_trace_data function with single dspy.Predict program."""

    # Define signature for string -> int conversion
    class StringToIntSignature(dspy.Signature):
        """Convert a string number to integer"""

        text: str = dspy.InputField()
        number: int = dspy.OutputField()

    # Create program with single dspy.Predict
    program = dspy.Predict(StringToIntSignature)

    # Create dummy dataset of size 5
    dataset = [
        Example(text="one", number=1).with_inputs("text"),
        Example(text="two", number=2).with_inputs("text"),
        Example(text="three", number=3).with_inputs("text"),
        Example(text="four", number=4).with_inputs("text"),
        Example(text="five", number=5).with_inputs("text"),
    ]

    # Define exact match metric
    def exact_match_metric(example, prediction, trace=None):
        return example.number == prediction.number

    # Configure dspy
    dspy.configure(lm=dspy.LM(engine="litellm", model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())

    # One reply per example: JSON parsing failures do not restart execution in
    # another format. Keep four successes and one captured parsing failure.
    responses = [
        ModelResponse(
            choices=[Choices(message=Message(content=(
                "This is an invalid JSON!" if number is None else f'```json\n{{"number": {number}}}\n```'
            )))],
            model="openai/gpt-4o-mini",
        )
        for number in (1, 2, None, 4, 5)
    ]

    with mock.patch("litellm.completion", side_effect=responses) as completion:
        # Call bootstrap_trace_data
        results = bootstrap_trace_data(
            program=program,
            dataset=dataset,
            metric=exact_match_metric,
            num_threads=1,
            raise_on_error=False,
            capture_failed_parses=True,
        )

    # Verify results, including the absence of a hidden JSON-mode replay.
    assert completion.call_count == 5
    assert len(results) == 5, f"Expected 5 results, got {len(results)}"

    # Count successful and failed predictions
    successful_count = 0
    failed_count = 0

    for result in results:
        assert "example" in result
        assert "prediction" in result
        assert "trace" in result
        assert "example_ind" in result
        assert "score" in result

        if isinstance(result["prediction"], FailedPrediction):
            failed_count += 1
            # Verify failed prediction structure
            assert hasattr(result["prediction"], "completion_text")
            assert hasattr(result["prediction"], "format_reward")
            assert result["prediction"].completion_text == "This is an invalid JSON!"
        else:
            successful_count += 1
            # Verify successful prediction structure
            assert hasattr(result["prediction"], "number")

    # Verify we have the expected number of successful and failed bootstrapping
    assert successful_count == 4, f"Expected 4 successful predictions, got {successful_count}"
    assert failed_count == 1, f"Expected 1 failed prediction, got {failed_count}"

    # Verify that traces are present
    for result in results:
        assert len(result["trace"]) > 0, "Trace should not be empty"
        # Each trace entry should be a tuple of (predictor, inputs, prediction)
        for trace_entry in result["trace"]:
            assert len(trace_entry) == 3, "Trace entry should have 3 elements"


@pytest.mark.parametrize("response, format_failure_score, expected_score", [
    ({}, -2, -2),
    ({"answer": "4"}, -2, -0.5),
    ({"answer": "4"}, -1, 0),
])
@pytest.mark.parametrize("adapter", [
    dspy.ChatAdapter(use_json_adapter_fallback=False),
    dspy.JSONAdapter(),
    dspy.XMLAdapter(use_json_adapter_fallback=False),
])
def test_chain_of_thought_parse_failure_is_captured(adapter, response, format_failure_score, expected_score):
    program = dspy.ChainOfThought("question -> answer")
    example = Example(question="2+2?", answer="4").with_inputs("question")
    lm = DummyLM([response], adapter=adapter)

    with dspy.context(lm=lm, adapter=adapter):
        data = bootstrap_trace_data(
            program=program,
            dataset=[example],
            metric=lambda e, p, t=None: 1.0,
            num_threads=1,
            raise_on_error=False,
            capture_failed_parses=True,
            failure_score=1,
            format_failure_score=format_failure_score,
        )

    assert len(data) == 1
    result = data[0]
    prediction = result["prediction"]
    assert isinstance(prediction, FailedPrediction)
    assert prediction.completion_text == lm.history[-1]["outputs"][0]
    assert prediction.format_reward == expected_score
    assert result["score"] == expected_score
    assert result["trace"] == [(program.predict, {"question": "2+2?"}, prediction)]


def test_bootstrap_trace_data_passes_callback_metadata(monkeypatch):
    from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module

    class DummyProgram(dspy.Module):
        def forward(self, **kwargs):  # pragma: no cover - stub forward
            return dspy.Prediction()

    captured_metadata: dict[str, Any] = {}

    class DummyEvaluate:
        def __init__(self, *args, **kwargs):
            pass

        def __call__(self, *args, callback_metadata=None, **kwargs):
            captured_metadata["value"] = callback_metadata

            class _Result:
                results: ClassVar[list[Any]] = []

            return _Result()

    monkeypatch.setattr(bootstrap_trace_module, "Evaluate", DummyEvaluate)

    bootstrap_trace_module.bootstrap_trace_data(
        program=DummyProgram(),
        dataset=[],
        callback_metadata={"disable_logging": True},
    )

    assert captured_metadata["value"] == {"disable_logging": True}


def test_capture_crashes_does_not_capture_lm_errors():
    """``capture_crashes`` converts program bugs into FailedPrediction, but an LMError is
    infrastructure: repainting it as a code failure would hand the optimizer an invalid
    evaluation, so it must propagate to the evaluator's error handling instead."""
    from dspy.utils.exceptions import LMRateLimitError

    example = Example(q="x").with_inputs("q")

    class Bug(dspy.Module):
        def forward(self, **kwargs):
            raise RuntimeError("a real code failure")

    data = bootstrap_trace_data(Bug(), dataset=[example], num_threads=1, capture_crashes=True)
    assert isinstance(data[0]["prediction"], FailedPrediction)
    assert "RuntimeError" in data[0]["prediction"].completion_text

    class Flaky(dspy.Module):
        def forward(self, **kwargs):
            raise LMRateLimitError("429 from the provider")

    # raise_on_error=False mirrors the flex GEPA call site (gepa_flex_utils).
    data = bootstrap_trace_data(
        Flaky(), dataset=[example], num_threads=1, capture_crashes=True, raise_on_error=False
    )
    assert data == []  # handled by the evaluator as an error, never repainted as a FailedPrediction