Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
9.9 KB
Raw Blame
import threading
from unittest.mock import Mock

import pytest

import dspy
from dspy import Signature
from dspy.evaluate.metrics import answer_exact_match
from dspy.predict import CodeAct
from dspy.primitives.code_interpreter import CodeInterpreterError
from dspy.utils import DummyLM
from tests.mock_interpreter import MockInterpreter, MockInterpreterFactory

pytestmark = pytest.mark.deno


class BasicQA(Signature):
    question = dspy.InputField()
    answer = dspy.OutputField(desc="often between 1 and 5 words")


class StaticPredictor:
    def __init__(self, **fields):
        self.fields = fields

    def __call__(self, **kwargs):
        return dspy.Prediction(**self.fields)


class RaisingPredictor:
    def __call__(self, **kwargs):
        raise ValueError("unexpected extractor failure")


def add(a: float, b: float) -> float:
    "add two numbers"
    return a + b


def test_codeact_warns_that_rlm_is_preferred():
    with pytest.warns(
        DeprecationWarning,
        match=r"CodeAct is deprecated and will be removed in 3\.5\. RLM is the preferred replacement\.",
    ):
        CodeAct(BasicQA, tools=[add])


def test_codeact_code_generation(pooled_interpreter):
    lm = DummyLM(
        [
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```",
                "finished": True,
            },
            {"plan": "Reason_B", "answer": "2"},
        ]
    )
    dspy.configure(lm=lm)
    program = CodeAct(BasicQA, tools=[add])
    res = program(pooled_interpreter, question="What is 1+1?")
    assert res.answer == "2"
    assert res.trajectory == {
        "code_output_0": '"2\\n"',
        "generated_code_0": "result = add(1,1)\nprint(result)",
    }


class ExtremumFinder(Signature):
    input_list = dspy.InputField()
    maximum = dspy.OutputField(desc="The maximum of the given numbers")
    minimum = dspy.OutputField(desc="The minimum of the given numbers")


def extract_maximum_minimum(input_list: str) -> dict[str, float]:
    numbers = list(map(float, input_list.split(",")))
    return {"maximum": max(numbers), "minimum": min(numbers)}


def test_codeact_support_multiple_fields(pooled_interpreter):
    lm = DummyLM(
        [
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nresult = extract_maximum_minimum('2, 3, 5, 6')\nprint(result)\n```",
                "finished": True,
            },
            {"plan": "Reason_B", "maximum": "6", "minimum": "2"},
        ]
    )
    dspy.configure(lm=lm)
    program = CodeAct(ExtremumFinder, tools=[extract_maximum_minimum])
    res = program(pooled_interpreter, input_list="2, 3, 5, 6")
    assert res.maximum == "6"
    assert res.minimum == "2"
    assert res.trajectory == {
        "code_output_0": "\"{'maximum': 6.0, 'minimum': 2.0}\\n\"",
        "generated_code_0": "result = extract_maximum_minimum('2, 3, 5, 6')\nprint(result)",
    }


def test_codeact_code_parse_failure(pooled_interpreter):
    lm = DummyLM(
        [
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nparse(error\n```",
                "finished": False,
            },
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```",
                "finished": True,
            },
            {"plan": "Reason_B", "answer": "2"},
        ]
    )
    dspy.configure(lm=lm)
    program = CodeAct(BasicQA, tools=[add])
    res = program(pooled_interpreter, question="What is 1+1?")
    assert res.answer == "2"
    assert res.trajectory == {
        "generated_code_0": "parse(error",
        "observation_0": "Failed to execute the generated code: Invalid Python syntax. message: ",
        "generated_code_1": "result = add(1,1)\nprint(result)",
        "code_output_1": '"2\\n"',
    }


def test_codeact_code_execution_failure(pooled_interpreter):
    lm = DummyLM(
        [
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nunknown+1\n```",
                "finished": False,
            },
            {
                "reasoning": "Reason_A",
                "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```",
                "finished": True,
            },
            {"plan": "Reason_B", "answer": "2"},
        ]
    )
    dspy.configure(lm=lm)
    program = CodeAct(BasicQA, tools=[add])
    res = program(pooled_interpreter, question="What is 1+1?")
    assert res.answer == "2"
    assert res.trajectory == {
        "generated_code_0": "unknown+1",
        "observation_0": "Failed to execute the generated code: NameError: [\"name 'unknown' is not defined\"]",
        "generated_code_1": "result = add(1,1)\nprint(result)",
        "code_output_1": '"2\\n"',
    }


def test_codeact_evaluate_creates_one_interpreter_per_example():
    tool_registration_barrier = threading.Barrier(4)

    def execute(code, variables):
        if code.startswith("def add"):
            tool_registration_barrier.wait(timeout=30)
            return ""
        return "2\n"

    factory = MockInterpreterFactory(execute_fn=execute)
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)
    program.codeact = StaticPredictor(generated_code="print(add(1, 1))", finished=True)
    program.extractor = StaticPredictor(answer="2")
    devset = [
        dspy.Example(question=f"What is 1+1? ({index})", answer="2").with_inputs("question") for index in range(4)
    ]

    result = dspy.Evaluate(
        devset=devset,
        metric=answer_exact_match,
        num_threads=4,
        display_progress=False,
    )(program)

    assert result.score == 100.0
    assert len(factory.instances) == 4
    assert len({id(interpreter) for interpreter in factory.instances}) == 4
    for interpreter in factory.instances:
        assert interpreter.call_count == 2
        with pytest.raises(CodeInterpreterError, match="shutdown"):
            interpreter.execute("print('closed')")


def test_codeact_factory_creates_fresh_interpreter_per_sequential_call():
    factory = MockInterpreterFactory(responses=["", "2\n"])
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)
    program.codeact = StaticPredictor(generated_code="print(add(1, 1))", finished=True)
    program.extractor = StaticPredictor(answer="2")

    first = program(question="What is 1+1?")
    second = program(question="What is 1+1 again?")

    assert first.answer == second.answer == "2"
    assert len(factory.instances) == 2
    assert factory.instances[0] is not factory.instances[1]
    for interpreter in factory.instances:
        with pytest.raises(CodeInterpreterError, match="shutdown"):
            interpreter.execute("print('closed')")


def test_codeact_allows_interpreter_as_signature_input():
    factory = MockInterpreterFactory(responses=["", "CPython\n"])
    program = CodeAct("interpreter -> answer", tools=[add], interpreter_factory=factory)
    program.codeact = Mock(return_value=dspy.Prediction(generated_code="print(interpreter)", finished=True))
    program.extractor = StaticPredictor(answer="CPython")

    result = program(interpreter="CPython")

    assert result.answer == "CPython"
    assert program.codeact.call_count == 1
    assert program.codeact.call_args.kwargs["interpreter"] == "CPython"


def test_codeact_rejects_keyword_interpreter_override():
    factory = MockInterpreterFactory()
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)

    with pytest.raises(TypeError, match="first positional argument"):
        program(question="What is 1+1?", interpreter=MockInterpreter())

    assert factory.instances == []


def test_codeact_rejects_removed_constructor_interpreter_keyword():
    with pytest.raises(TypeError, match="unexpected keyword argument 'interpreter'"):
        CodeAct(BasicQA, tools=[add], interpreter=MockInterpreter())


def test_codeact_does_not_shutdown_caller_owned_interpreter():
    factory = MockInterpreterFactory()
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)
    program.codeact = StaticPredictor(generated_code="print(add(1, 1))", finished=True)
    program.extractor = StaticPredictor(answer="2")
    interpreter = MockInterpreter(responses=["", "2\n"])

    try:
        result = program(interpreter, question="What is 1+1?")

        assert result.answer == "2"
        assert factory.instances == []
        assert interpreter.execute("print('still open')") == ""
    finally:
        interpreter.shutdown()


def test_codeact_shuts_down_factory_interpreter_when_extractor_raises():
    factory = MockInterpreterFactory(responses=["", "2\n"])
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)
    program.codeact = StaticPredictor(generated_code="print(add(1, 1))", finished=True)
    program.extractor = RaisingPredictor()

    with pytest.raises(ValueError, match="unexpected extractor failure"):
        program(question="What is 1+1?")

    assert len(factory.instances) == 1
    with pytest.raises(CodeInterpreterError, match="shutdown"):
        factory.instances[0].execute("print('closed')")


def test_codeact_propagates_terminal_interpreter_failure_and_shuts_down():
    factory = MockInterpreterFactory(responses=["", CodeInterpreterError("protocol corrupt")])
    program = CodeAct(BasicQA, tools=[add], interpreter_factory=factory)
    program.codeact = StaticPredictor(generated_code="print(add(1, 1))", finished=True)

    with pytest.raises(CodeInterpreterError, match="protocol corrupt"):
        program(question="What is 1+1?")

    assert len(factory.instances) == 1
    with pytest.raises(CodeInterpreterError, match="shutdown"):
        factory.instances[0].execute("print('closed')")


class CustomTool:
    def __call__(self, a: float, b: float) -> float:
        return a + b


def test_codeact_tool_validation():
    with pytest.raises(ValueError, match=r"CodeAct only accepts functions and not callable objects."):
        CodeAct(BasicQA, tools=[CustomTool()])