Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
12.4 KB
Raw Blame
import asyncio
from unittest.mock import Mock, patch

import cloudpickle
import pytest

import dspy
from dspy.teleprompt.bettertogether import BetterTogether
from dspy.teleprompt.teleprompt import Teleprompter
from dspy.teleprompt.vanilla import LabeledFewShot
from dspy.utils.callback import ACTIVE_CALL_ID, BaseCallback
from dspy.utils.dummies import DummyLM


class RecordingCallback(BaseCallback):
    def __init__(self):
        self.events = []

    def on_compile_start(self, call_id, instance, inputs):
        self.events.append({
            "handler": "compile_start",
            "call_id": call_id,
            "parent_call_id": ACTIVE_CALL_ID.get(),
            "instance": instance,
            "inputs": inputs,
        })

    def on_compile_end(self, call_id, outputs, exception):
        self.events.append({
            "handler": "compile_end",
            "call_id": call_id,
            "parent_call_id": ACTIVE_CALL_ID.get(),
            "outputs": outputs,
            "exception": exception,
        })

    def on_module_start(self, call_id, instance, inputs):
        self.events.append({
            "handler": "module_start",
            "call_id": call_id,
            "parent_call_id": ACTIVE_CALL_ID.get(),
            "instance": instance,
            "inputs": inputs,
        })

    def on_module_end(self, call_id, outputs, exception):
        self.events.append({
            "handler": "module_end",
            "call_id": call_id,
            "parent_call_id": ACTIVE_CALL_ID.get(),
            "outputs": outputs,
            "exception": exception,
        })


def events_for(callback, handler):
    return [event for event in callback.events if event["handler"] == handler]


def test_compile_callbacks_report_bound_inputs_output_and_call_id():
    class Optimizer(Teleprompter):
        def compile(self, student, *, trainset, teacher=None):
            return student

    callback = RecordingCallback()
    optimizer = Optimizer()
    student = dspy.Module()
    trainset = [dspy.Example(question="test").with_inputs("question")]

    with dspy.context(callbacks=[callback]):
        result = optimizer.compile(student, trainset=trainset)

    start, end = callback.events
    assert start["handler"] == "compile_start"
    assert start["instance"] is optimizer
    assert start["inputs"] == {"student": student, "trainset": trainset, "teacher": None}
    assert start["parent_call_id"] is None
    assert end["handler"] == "compile_end"
    assert end["call_id"] == start["call_id"]
    assert end["outputs"] is result is student
    assert end["exception"] is None


def test_compile_callback_reports_and_propagates_same_exception():
    expected_error = ValueError("compile failed")

    class FailingOptimizer(Teleprompter):
        def __init__(self):
            self.should_fail = True

        def compile(self, student, *, trainset):
            if self.should_fail:
                self.should_fail = False
                raise expected_error
            return student

    callback = RecordingCallback()
    optimizer = FailingOptimizer()
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        with pytest.raises(ValueError) as exc_info:
            optimizer.compile(student, trainset=[])
        assert optimizer.compile(student, trainset=[]) is student

    first_start, first_end, second_start, second_end = callback.events
    assert first_end["call_id"] == first_start["call_id"]
    assert first_end["outputs"] is None
    assert first_end["exception"] is expected_error
    assert exc_info.value is expected_error
    assert second_end["call_id"] == second_start["call_id"]
    assert second_end["outputs"] is student
    assert second_end["exception"] is None


def test_nested_optimizer_and_module_callbacks_preserve_hierarchy():
    class Student(dspy.Module):
        def forward(self, value):
            return dspy.Prediction(value=value)

    class InnerOptimizer(Teleprompter):
        def compile(self, student, *, trainset):
            student(value=trainset[0])
            return student

    class OuterOptimizer(Teleprompter):
        def compile(self, student, *, trainset, optimizer):
            return optimizer.compile(student, trainset=trainset)

    callback = RecordingCallback()
    outer = OuterOptimizer()
    inner = InnerOptimizer()
    student = Student()

    with dspy.context(callbacks=[callback]):
        assert outer.compile(student, trainset=["value"], optimizer=inner) is student

    assert [event["handler"] for event in callback.events] == [
        "compile_start",
        "compile_start",
        "module_start",
        "module_end",
        "compile_end",
        "compile_end",
    ]
    outer_start, inner_start, module_start, module_end, inner_end, outer_end = callback.events
    assert inner_start["parent_call_id"] == outer_start["call_id"]
    assert module_start["parent_call_id"] == inner_start["call_id"]
    assert module_end["parent_call_id"] == inner_start["call_id"]
    assert inner_end["parent_call_id"] == outer_start["call_id"]
    assert outer_end["parent_call_id"] is None
    assert len({outer_start["call_id"], inner_start["call_id"], module_start["call_id"]}) == 3


def test_inherited_compile_is_not_wrapped_twice_and_override_is_wrapped_once():
    class BaseOptimizer(Teleprompter):
        def compile(self, student, *, trainset):
            return student

    class InheritedOptimizer(BaseOptimizer):
        pass

    class OverridingOptimizer(BaseOptimizer):
        def compile(self, student, *, trainset):
            return super().compile(student, trainset=trainset)

    callback = RecordingCallback()
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        InheritedOptimizer().compile(student, trainset=[])
        OverridingOptimizer().compile(student, trainset=[])

    starts = events_for(callback, "compile_start")
    ends = events_for(callback, "compile_end")
    assert len(starts) == len(ends) == 2
    assert isinstance(starts[0]["instance"], InheritedOptimizer)
    assert isinstance(starts[1]["instance"], OverridingOptimizer)
    assert starts[1]["parent_call_id"] is None


@pytest.mark.asyncio
async def test_async_override_calling_super_emits_one_compile_callback_pair():
    class BaseOptimizer(Teleprompter):
        async def compile(self, student, *, trainset):
            return student

    class OverridingOptimizer(BaseOptimizer):
        async def compile(self, student, *, trainset):
            return await super().compile(student, trainset=trainset)

    callback = RecordingCallback()
    optimizer = OverridingOptimizer()
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        assert await optimizer.compile(student, trainset=[]) is student

    start, end = callback.events
    assert start["instance"] is optimizer
    assert start["parent_call_id"] is None
    assert end["call_id"] == start["call_id"]


@pytest.mark.asyncio
async def test_same_instance_compile_in_child_task_emits_nested_callback_pair():
    class Optimizer(Teleprompter):
        async def compile(self, student, *, trainset, child=False):
            if not child:
                return await asyncio.create_task(self.compile(student, trainset=trainset, child=True))
            return student

    callback = RecordingCallback()
    optimizer = Optimizer()
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        assert await optimizer.compile(student, trainset=[]) is student

    outer_start, child_start, child_end, outer_end = callback.events
    assert [event["handler"] for event in callback.events] == [
        "compile_start",
        "compile_start",
        "compile_end",
        "compile_end",
    ]
    assert outer_start["instance"] is child_start["instance"] is optimizer
    assert outer_start["parent_call_id"] is None
    assert child_start["parent_call_id"] == outer_start["call_id"]
    assert child_end["call_id"] == child_start["call_id"]
    assert child_end["parent_call_id"] == outer_start["call_id"]
    assert outer_end["call_id"] == outer_start["call_id"]
    assert outer_end["parent_call_id"] is None


def test_compile_callback_wrapper_round_trips_through_cloudpickle():
    class Optimizer(Teleprompter):
        def compile(self, student, *, trainset):
            return student

    optimizer = cloudpickle.loads(cloudpickle.dumps(Optimizer))()
    callback = RecordingCallback()
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        assert optimizer.compile(student, trainset=[]) is student

    start, end = callback.events
    assert start["instance"] is optimizer
    assert end["call_id"] == start["call_id"]


def test_existing_optimizer_compile_is_instrumented():
    callback = RecordingCallback()
    optimizer = LabeledFewShot(k=0)
    student = dspy.Module()

    with dspy.context(callbacks=[callback]):
        compiled = optimizer.compile(student, trainset=[])

    start, end = callback.events
    assert start["instance"] is optimizer
    assert start["inputs"] == {"student": student, "trainset": [], "sample": True}
    assert end["outputs"] is compiled
    assert end["exception"] is None


def test_bettertogether_nests_child_compile_callback():
    class Student(dspy.Module):
        def __init__(self):
            super().__init__()
            self.predictor = dspy.Predict("question -> answer")

        def forward(self, question):
            return self.predictor(question=question)

    class ChildOptimizer(Teleprompter):
        def compile(self, student, *, trainset):
            return student

    callback = RecordingCallback()
    child = ChildOptimizer()
    optimizer = BetterTogether(metric=lambda example, prediction, trace=None: 1.0, child=child)
    student = Student()
    student.set_lm(DummyLM([{"answer": "answer"}]))
    trainset = [dspy.Example(question="question", answer="answer").with_inputs("question")]

    with (
        patch("dspy.teleprompt.bettertogether.eval_candidate_program", return_value=Mock(score=1.0)),
        patch("dspy.teleprompt.bettertogether.launch_lms"),
        patch("dspy.teleprompt.bettertogether.kill_lms"),
        dspy.context(callbacks=[callback]),
    ):
        optimizer.compile(student, trainset=trainset, valset=trainset, strategy="child")

    starts = events_for(callback, "compile_start")
    ends = events_for(callback, "compile_end")
    assert len(starts) == len(ends) == 2
    assert starts[0]["instance"] is optimizer
    assert starts[0]["parent_call_id"] is None
    assert starts[1]["instance"] is child
    assert starts[1]["parent_call_id"] == starts[0]["call_id"]
    assert ends[0]["call_id"] == starts[1]["call_id"]
    assert ends[0]["parent_call_id"] == starts[0]["call_id"]
    assert ends[1]["call_id"] == starts[0]["call_id"]
    assert ends[1]["parent_call_id"] is None


def test_bettertogether_reports_child_failure_but_completes_outer_run():
    expected_error = RuntimeError("child compile failed")

    class Student(dspy.Module):
        def __init__(self):
            super().__init__()
            self.predictor = dspy.Predict("question -> answer")

        def forward(self, question):
            return self.predictor(question=question)

    class FailingChildOptimizer(Teleprompter):
        def compile(self, student, *, trainset):
            raise expected_error

    callback = RecordingCallback()
    child = FailingChildOptimizer()
    optimizer = BetterTogether(metric=lambda example, prediction, trace=None: 1.0, child=child)
    student = Student()
    student.set_lm(DummyLM([{"answer": "answer"}]))
    trainset = [dspy.Example(question="question", answer="answer").with_inputs("question")]

    with (
        patch("dspy.teleprompt.bettertogether.eval_candidate_program", return_value=Mock(score=1.0)),
        patch("dspy.teleprompt.bettertogether.launch_lms"),
        patch("dspy.teleprompt.bettertogether.kill_lms"),
        dspy.context(callbacks=[callback]),
    ):
        compiled = optimizer.compile(student, trainset=trainset, valset=trainset, strategy="child")

    starts = events_for(callback, "compile_start")
    ends = events_for(callback, "compile_end")
    assert len(starts) == len(ends) == 2
    assert starts[1]["instance"] is child
    assert starts[1]["parent_call_id"] == starts[0]["call_id"]
    assert ends[0]["call_id"] == starts[1]["call_id"]
    assert ends[0]["outputs"] is None
    assert ends[0]["exception"] is expected_error
    assert ends[1]["call_id"] == starts[0]["call_id"]
    assert ends[1]["outputs"] is compiled
    assert ends[1]["exception"] is None
    assert compiled.flag_compilation_error_occurred is True