Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
11.1 KB
Raw Blame
import asyncio
import time

import cloudpickle
import pytest

import dspy
from dspy.utils.callback import ACTIVE_CALL_ID, BaseCallback, with_callbacks
from dspy.utils.callback_context import _bind_active_call_id
from dspy.utils.dummies import DummyLM


@pytest.fixture(autouse=True)
def reset_settings():
    # Make sure the settings are reset after each test
    original_settings = dspy.settings.copy()

    yield

    dspy.configure(**original_settings)


class MyCallback(BaseCallback):
    """A simple callback that records the calls."""

    def __init__(self):
        self.calls = []

    def on_module_start(self, call_id, instance, inputs):
        self.calls.append({"handler": "on_module_start", "instance": instance, "inputs": inputs})

    def on_module_end(self, call_id, outputs, exception):
        self.calls.append({"handler": "on_module_end", "outputs": outputs, "exception": exception})

    def on_lm_start(self, call_id, instance, inputs):
        self.calls.append({"handler": "on_lm_start", "instance": instance, "inputs": inputs})

    def on_lm_end(self, call_id, outputs, exception):
        self.calls.append({"handler": "on_lm_end", "outputs": outputs, "exception": exception})

    def on_adapter_format_start(self, call_id, instance, inputs):
        self.calls.append({"handler": "on_adapter_format_start", "instance": instance, "inputs": inputs})

    def on_adapter_format_end(self, call_id, outputs, exception):
        self.calls.append({"handler": "on_adapter_format_end", "outputs": outputs, "exception": exception})

    def on_adapter_parse_start(self, call_id, instance, inputs):
        self.calls.append({"handler": "on_adapter_parse_start", "instance": instance, "inputs": inputs})

    def on_adapter_parse_end(self, call_id, outputs, exception):
        self.calls.append({"handler": "on_adapter_parse_end", "outputs": outputs, "exception": exception})

    def on_tool_start(self, call_id, instance, inputs):
        self.calls.append({"handler": "on_tool_start", "instance": instance, "inputs": inputs})

    def on_tool_end(self, call_id, outputs, exception):
        self.calls.append({"handler": "on_tool_end", "outputs": outputs, "exception": exception})


def test_bind_active_call_id_restores_context_after_exception():
    observed_call_ids = []

    def fail():
        observed_call_ids.append(ACTIVE_CALL_ID.get())
        raise ValueError("boom")

    token = ACTIVE_CALL_ID.set("parent-call")
    try:
        bound = _bind_active_call_id(fail)
    finally:
        ACTIVE_CALL_ID.reset(token)

    with pytest.raises(ValueError, match="boom"):
        bound()

    assert observed_call_ids == ["parent-call"]
    assert ACTIVE_CALL_ID.get() is None


@pytest.mark.asyncio
async def test_with_callbacks_wrappers_round_trip_through_cloudpickle():
    async def async_fn(self):
        return "async"

    sync_fn = cloudpickle.loads(cloudpickle.dumps(with_callbacks(lambda self: "sync")))
    async_fn = cloudpickle.loads(cloudpickle.dumps(with_callbacks(async_fn)))

    assert sync_fn(object()) == "sync"
    assert await async_fn(object()) == "async"


@pytest.mark.parametrize(
    ("args", "kwargs"),
    [
        ([1, "2", 3.0], {}),
        ([1, "2"], {"z": 3.0}),
        ([1], {"y": "2", "z": 3.0}),
        ([], {"x": 1, "y": "2", "z": 3.0}),
    ],
)
def test_callback_injection(args, kwargs):
    class Target(dspy.Module):
        @with_callbacks
        def forward(self, x: int, y: str, z: float) -> int:
            time.sleep(0.1)
            return x + int(y) + int(z)

    callback = MyCallback()
    dspy.configure(callbacks=[callback])

    target = Target()
    result = target.forward(*args, **kwargs)

    assert result == 6

    assert len(callback.calls) == 2
    assert callback.calls[0]["handler"] == "on_module_start"
    assert callback.calls[0]["inputs"] == {"x": 1, "y": "2", "z": 3.0}
    assert callback.calls[1]["handler"] == "on_module_end"
    assert callback.calls[1]["outputs"] == 6


def test_callback_injection_local():
    class Target(dspy.Module):
        @with_callbacks
        def forward(self, x: int, y: str, z: float) -> int:
            time.sleep(0.1)
            return x + int(y) + int(z)

    callback = MyCallback()

    target_1 = Target(callbacks=[callback])
    result = target_1.forward(1, "2", 3.0)

    assert result == 6

    assert len(callback.calls) == 2
    assert callback.calls[0]["handler"] == "on_module_start"
    assert callback.calls[0]["inputs"] == {"x": 1, "y": "2", "z": 3.0}
    assert callback.calls[1]["handler"] == "on_module_end"
    assert callback.calls[1]["outputs"] == 6

    callback.calls = []

    target_2 = Target()
    result = target_2.forward(1, "2", 3.0)

    # Other instance should not trigger the callback
    assert not callback.calls


def test_callback_error_handling():
    expected_error = ValueError("Error")

    class Target(dspy.Module):
        @with_callbacks
        def forward(self, x: int, y: str, z: float) -> int:
            time.sleep(0.1)
            raise expected_error

    callback = MyCallback()
    dspy.configure(callbacks=[callback])

    target = Target()

    with pytest.raises(ValueError, match="Error") as exc_info:
        target.forward(1, "2", 3.0)

    assert len(callback.calls) == 2
    assert callback.calls[0]["handler"] == "on_module_start"
    assert callback.calls[1]["handler"] == "on_module_end"
    assert callback.calls[1]["exception"] is exc_info.value is expected_error


@pytest.mark.parametrize("expected_error", [KeyboardInterrupt(), SystemExit()])
def test_callback_reports_and_propagates_base_exception(expected_error):
    class Target(dspy.Module):
        @with_callbacks
        def forward(self):
            raise expected_error

    callback = MyCallback()
    target = Target(callbacks=[callback])

    with pytest.raises(BaseException) as exc_info:
        target.forward()

    assert callback.calls[1]["handler"] == "on_module_end"
    assert callback.calls[1]["outputs"] is None
    assert callback.calls[1]["exception"] is exc_info.value is expected_error
    assert ACTIVE_CALL_ID.get() is None


@pytest.mark.asyncio
async def test_callback_reports_and_propagates_async_cancellation():
    started = asyncio.Event()

    class Target(dspy.Module):
        @with_callbacks
        async def run(self):
            started.set()
            await asyncio.Event().wait()

    callback = MyCallback()
    target = Target(callbacks=[callback])
    task = asyncio.create_task(target.run())
    await started.wait()
    task.cancel()

    with pytest.raises(asyncio.CancelledError):
        await task

    assert callback.calls[1]["handler"] == "on_module_end"
    assert callback.calls[1]["outputs"] is None
    assert isinstance(callback.calls[1]["exception"], asyncio.CancelledError)
    assert ACTIVE_CALL_ID.get() is None


def test_multiple_callbacks():
    class Target(dspy.Module):
        @with_callbacks
        def forward(self, x: int, y: str, z: float) -> int:
            time.sleep(0.1)
            return x + int(y) + int(z)

    callback_1 = MyCallback()
    callback_2 = MyCallback()
    dspy.configure(callbacks=[callback_1, callback_2])

    target = Target()
    result = target.forward(1, "2", 3.0)

    assert result == 6

    assert len(callback_1.calls) == 2
    assert len(callback_2.calls) == 2


def test_callback_complex_module():
    callback = MyCallback()
    dspy.configure(
        lm=DummyLM({"How are you?": {"answer": "test output", "plan": "No more responses"}}),
        callbacks=[callback],
    )

    cot = dspy.ChainOfThought("question -> answer", n=3)
    result = cot(question="How are you?")
    assert result["answer"] == "test output"
    assert result["reasoning"] == "No more responses"

    assert len(callback.calls) == 14
    assert [call["handler"] for call in callback.calls] == [
        "on_module_start",
        "on_module_start",
        "on_adapter_format_start",
        "on_adapter_format_end",
        "on_lm_start",
        "on_lm_end",
        # Parsing will run per output (n=3)
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_module_end",
        "on_module_end",
    ]

@pytest.mark.asyncio
async def test_callback_async_module():
    callback = MyCallback()
    with dspy.context(
        lm=DummyLM({"How are you?": {"answer": "test output", "plan": "No more responses"}}),
        callbacks=[callback],
    ):
        cot = dspy.ChainOfThought("question -> answer", n=3)
        result = await cot.acall(question="How are you?")
    assert result["answer"] == "test output"
    assert result["reasoning"] == "No more responses"

    assert len(callback.calls) == 14
    assert [call["handler"] for call in callback.calls] == [
        "on_module_start",
        "on_module_start",
        "on_adapter_format_start",
        "on_adapter_format_end",
        "on_lm_start",
        "on_lm_end",
        # Parsing will run per output (n=3)
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_adapter_parse_start",
        "on_adapter_parse_end",
        "on_module_end",
        "on_module_end",
    ]


def test_tool_calls():
    callback = MyCallback()
    dspy.configure(callbacks=[callback])

    def tool_1(query: str) -> str:
        """A dummy tool function."""
        return "result 1"

    def tool_2(query: str) -> str:
        """Another dummy tool function."""
        return "result 2"

    class MyModule(dspy.Module):
        def __init__(self):
            self.tools = [dspy.Tool(tool_1), dspy.Tool(tool_2)]

        def forward(self, query: str) -> str:
            query = self.tools[0](query=query)
            return self.tools[1](query=query)

    module = MyModule()
    result = module("query")

    assert result == "result 2"
    assert len(callback.calls) == 6
    assert [call["handler"] for call in callback.calls] == [
        "on_module_start",
        "on_tool_start",
        "on_tool_end",
        "on_tool_start",
        "on_tool_end",
        "on_module_end",
    ]


def test_active_id():
    # Test the call ID is generated and handled properly
    class CustomCallback(BaseCallback):
        def __init__(self):
            self.parent_call_ids = []
            self.call_ids = []

        def on_module_start(self, call_id, instance, inputs):
            parent_call_id = ACTIVE_CALL_ID.get()
            self.parent_call_ids.append(parent_call_id)
            self.call_ids.append(call_id)

    class Parent(dspy.Module):
        def __init__(self):
            self.child_1 = Child()
            self.child_2 = Child()

        def forward(self):
            self.child_1()
            self.child_2()

    class Child(dspy.Module):
        def forward(self):
            pass

    callback = CustomCallback()
    dspy.configure(callbacks=[callback])

    parent = Parent()
    parent()

    assert len(callback.call_ids) == 3
    # All three calls should have different call ids
    assert len(set(callback.call_ids)) == 3
    parent_call_id = callback.call_ids[0]
    assert callback.parent_call_ids == [None, parent_call_id, parent_call_id]