Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
12.5 KB
Raw Blame
import contextvars
import os
import tempfile
import threading
import time
from pathlib import Path

import pytest

import dspy
from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreterError, FinalOutput
from dspy.utils.callback import BaseCallback
from dspy.utils.dummies import DummyLM


def test_persistent_worker_is_separate_and_captures_stdout(capsys):
    with dspy.LocalInterpreter() as interpreter:
        assert interpreter.execute("import os\nos.getpid()") != os.getpid()
        interpreter.execute("remembered = 41")
        assert interpreter.execute("remembered + 1") == 42
        assert interpreter.execute("print('guest only')") == "guest only"
        assert capsys.readouterr().out == ""


def test_os_level_output_does_not_corrupt_protocol():
    with dspy.LocalInterpreter() as interpreter:
        output = interpreter.execute(
            "import os, subprocess, sys\n"
            "os.write(1, b'guest fd stdout\\n')\n"
            "os.write(2, b'guest fd stderr\\n')\n"
            "completed = subprocess.run([sys.executable, '-c', \"import os; os.write(1, b'child stdout\\\\n')\"])"
        )
        assert output == "guest fd stdout\nguest fd stderr\nchild stdout"
        assert interpreter.execute("6 * 7") == 42


def test_python_stdout_reassignment_does_not_break_later_capture():
    with dspy.LocalInterpreter() as interpreter:
        assert interpreter.execute("import io, sys\nsys.stdout = io.StringIO()\nprint('hidden')") is None
        assert interpreter.execute("print('captured')") == "captured"


def test_background_thread_is_terminal_before_it_can_cross_executions():
    calls = []
    interpreter = dspy.LocalInterpreter(tools={"record": lambda *, value: calls.append(value)})

    with pytest.raises(CodeInterpreterError, match="left background threads running"):
        interpreter.execute(
            "import threading\n"
            "def late():\n"
            "    threading.Event().wait(60)\n"
            "    record(value='late')\n"
            "threading.Thread(target=late, daemon=True).start()"
        )

    assert calls == []
    with pytest.raises(CodeInterpreterError, match="shut down"):
        interpreter.execute("1")


def test_shutdown_terminates_descendant_processes():
    fd, path_string = tempfile.mkstemp(prefix="local-interpreter-child-", suffix=".txt")
    os.close(fd)
    path = Path(path_string)
    path.unlink()
    interpreter = dspy.LocalInterpreter()

    try:
        assert interpreter.execute(
            "import subprocess, sys\n"
            f"path = {str(path)!r}\n"
            "child = subprocess.Popen([sys.executable, '-c', "
            "f\"import time; from pathlib import Path; time.sleep(0.4); Path({path!r}).write_text('survived')\"])\n"
            "'child started'"
        ) == "child started"
        interpreter.shutdown()
        time.sleep(0.8)
        assert not path.exists()
    finally:
        interpreter.shutdown()
        path.unlink(missing_ok=True)


def test_tools_and_typed_submit():
    fields = [{"name": "answer", "type": "str"}, {"name": "score", "type": "int"}]
    with dspy.LocalInterpreter(
        tools={"add": lambda *, left, right: left + right}, output_fields=fields
    ) as interpreter:
        assert interpreter.execute("add(left=19, right=23)") == 42
        result = interpreter.execute("SUBMIT(answer='yes', score=42)")
        assert isinstance(result, FinalOutput)
        assert result.output == {"answer": "yes", "score": 42}


@pytest.mark.parametrize("name", ["not-valid", "class", "SUBMIT", "__builtins__"])
def test_invalid_tool_names_fail_without_starting_or_consuming_session(name):
    with pytest.raises(ValueError, match="invalid names"):
        dspy.LocalInterpreter(tools={name: lambda: 1})

    interpreter = dspy.LocalInterpreter()
    interpreter.tools[name] = lambda: 1
    with pytest.raises(ValueError, match="invalid names"):
        interpreter.execute("1")
    assert interpreter._process is None

    interpreter.tools = {}
    assert interpreter.execute("6 * 7") == 42
    interpreter.shutdown()


@pytest.mark.parametrize(
    "output_fields",
    [
        [{"name": "class"}],
        [{"name": "not-valid"}],
        [{"name": "answer"}, {"name": "answer"}],
        [{"type": "str"}],
        [{"name": "answer", "default": object()}],
    ],
)
def test_invalid_output_fields_fail_without_consuming_session(output_fields):
    with pytest.raises(ValueError):
        dspy.LocalInterpreter(output_fields=output_fields)

    interpreter = dspy.LocalInterpreter()
    assert interpreter.execute("remembered = 41") is None
    process = interpreter._process
    interpreter.output_fields = output_fields
    with pytest.raises(ValueError):
        interpreter.execute("1")
    assert interpreter._process is process

    interpreter.output_fields = [{"name": "answer"}]
    assert interpreter.execute("remembered + 1") == 42
    assert interpreter.execute("SUBMIT(answer=42)").output == {"answer": 42}
    interpreter.shutdown()


def test_reserved_variable_names_fail_without_starting_or_consuming_session():
    interpreter = dspy.LocalInterpreter(tools={"add": lambda left, right: left + right})
    for variables in ({"class": 1}, {"SUBMIT": "shadowed"}, {"__builtins__": {}}, {"add": "shadowed"}):
        with pytest.raises(CodeInterpreterError, match="invalid names"):
            interpreter.execute("1", variables=variables)
        assert interpreter._process is None

    assert interpreter.execute("len([1])") == 1
    assert interpreter.execute("add(19, 23)") == 42
    assert interpreter.execute("SUBMIT(42)").output == {"output": 42}
    interpreter.shutdown()


def test_guest_cannot_corrupt_interpreter_owned_globals_across_executions():
    with dspy.LocalInterpreter(tools={"add": lambda left, right: left + right}) as interpreter:
        interpreter.execute("__builtins__['len'] = None\nSUBMIT = None\nadd = None")
        assert interpreter.execute("len([1])") == 1
        assert interpreter.execute("add(19, 23)") == 42
        assert interpreter.execute("SUBMIT(42)").output == {"output": 42}
        with pytest.raises(CodeExecutionError, match="_builtins"):
            interpreter.execute("SUBMIT.__self__._builtins['len'] = None")
        assert interpreter.execute("len([1])") == 1


def test_errors_are_recoverable_but_timeout_is_terminal():
    interpreter = dspy.LocalInterpreter(execution_timeout=1)
    with pytest.raises(SyntaxError):
        interpreter.execute("if")
    with pytest.raises(CodeExecutionError, match="ZeroDivisionError"):
        interpreter.execute("1 / 0")
    assert interpreter.execute("6 * 7") == 42
    with pytest.raises(CodeInterpreterError, match="exceeded execution timeout"):
        interpreter.execute("import time\ntime.sleep(10)")
    with pytest.raises(CodeInterpreterError, match="shut down"):
        interpreter.execute("1")
    interpreter.shutdown()


def test_async_host_tool():
    async def add(*, left, right):
        return left + right

    with dspy.LocalInterpreter(tools={"add": add}) as interpreter:
        assert interpreter.execute("add(left=19, right=23)") == 42


def test_timed_async_host_tool_preserves_context():
    marker = contextvars.ContextVar("marker", default="missing")
    marker.set("visible")

    async def read_marker():
        return marker.get()

    with dspy.LocalInterpreter(tools={"read_marker": read_marker}, execution_timeout=1) as interpreter:
        assert interpreter.execute("read_marker()") == "visible"


def test_host_tool_within_execution_timeout():
    def slow():
        time.sleep(0.08)
        return 42

    with dspy.LocalInterpreter(tools={"slow": slow}, execution_timeout=1) as interpreter:
        assert interpreter.execute("slow()") == 42


def test_parallel_guest_code_invokes_host_tools_concurrently_and_routes_responses():
    rendezvous = threading.Barrier(2)

    def echo(value):
        rendezvous.wait(timeout=1)
        if value == "slow":
            time.sleep(0.05)
        return value

    code = """
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=2) as pool:
    futures = [pool.submit(echo, value) for value in ["slow", "fast"]]
[future.result() for future in futures]
"""
    with dspy.LocalInterpreter(tools={"echo": echo}, execution_timeout=2) as interpreter:
        assert interpreter.execute(code) == ["slow", "fast"]


def test_execution_timeout_does_not_wait_for_blocked_host_tool():
    tool_started = threading.Event()
    release_tool = threading.Event()

    def blocked():
        tool_started.set()
        release_tool.wait(timeout=10)
        return 42

    interpreter = dspy.LocalInterpreter(tools={"blocked": blocked}, execution_timeout=1)
    interpreter.start()
    started = time.monotonic()
    with pytest.raises(CodeInterpreterError, match="exceeded execution timeout"):
        interpreter.execute("blocked()")
    elapsed = time.monotonic() - started
    release_tool.set()

    assert tool_started.wait(timeout=5)
    assert elapsed < 5
    with pytest.raises(CodeInterpreterError, match="shut down"):
        interpreter.execute("1")


def test_interpreter_callbacks():
    events = []

    class RecordingCallback(BaseCallback):
        def on_interpreter_execute_start(self, call_id, instance, inputs):
            events.append(("execute_start", inputs["code"]))

        def on_interpreter_execute_end(self, call_id, outputs, exception=None):
            events.append(("execute_end", outputs))

        def on_interpreter_tool_call_start(self, call_id, instance, inputs):
            events.append(("tool_start", inputs["tool_name"]))

        def on_interpreter_tool_call_end(self, call_id, outputs, exception=None):
            events.append(("tool_end", outputs))

    with dspy.LocalInterpreter(tools={"answer": lambda: 42}, callbacks=[RecordingCallback()]) as interpreter:
        assert interpreter.execute("answer()") == 42

    assert ("execute_start", "answer()") in events
    assert ("tool_start", "answer") in events
    assert ("tool_end", 42) in events
    assert ("execute_end", 42) in events


def test_rlm_uses_local_interpreter():
    calls = []

    def add(*, left: int, right: int) -> int:
        calls.append((left, right))
        return left + right

    class Actions(dspy.Predict):
        def __init__(self, signature):
            super().__init__(signature)
            self.actions = iter(
                [
                    dspy.Prediction(reasoning="calculate", code="total = add(left=19, right=23)"),
                    dspy.Prediction(reasoning="submit", code="SUBMIT(answer=str(total))"),
                ]
            )

        def forward(self, **kwargs):
            return next(self.actions)

    rlm = dspy.RLM(
        "question: str -> answer: str",
        max_iters=2,
        tools=[add],
        interpreter_factory=dspy.LocalInterpreter,
    )
    rlm.generate_action = Actions(rlm.generate_action.signature)

    assert rlm(question="What is 19 + 23?").answer == "42"
    assert calls == [(19, 23)]
    assert dspy.LocalInterpreter.execution_instructions in rlm.generate_action.signature.instructions


@pytest.mark.asyncio
async def test_async_rlm_uses_local_interpreter_with_async_tool():
    async def add(*, left: int, right: int) -> int:
        return left + right

    class Actions(dspy.Predict):
        def __init__(self, signature):
            super().__init__(signature)
            self.actions = iter(
                [
                    dspy.Prediction(reasoning="calculate", code="total = add(left=19, right=23)"),
                    dspy.Prediction(reasoning="submit", code="SUBMIT(answer=str(total))"),
                ]
            )

        async def aforward(self, **kwargs):
            return next(self.actions)

    rlm = dspy.RLM(
        "question: str -> answer: str",
        max_iters=2,
        tools=[add],
        interpreter_factory=dspy.LocalInterpreter,
    )
    rlm.generate_action = Actions(rlm.generate_action.signature)

    assert (await rlm.acall(question="What is 19 + 23?")).answer == "42"


def test_flex_uses_local_interpreter():
    class Signature(dspy.Signature):
        value: int = dspy.InputField()
        result: int = dspy.OutputField()

    module_src = """
class AddModule(dspy.Module):
    def __init__(self):
        super().__init__()
        self.solve = dspy.Predict("value: int -> result: int")

    def forward(self, **inputs):
        predicted = self.solve(value=inputs["value"])
        return dspy.Prediction(result=predicted.result)
""".strip()

    program = dspy.Flex(Signature, interpreter_factory=dspy.LocalInterpreter)
    program._bind_code(module_src)
    with dspy.context(lm=DummyLM([{"result": 42}])):
        assert program(value=20).result == 42