Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
12.1 KB
Raw Blame
"""dspy.Flex accepts tools, hands them to the underlying RLM baseline, exposes them to the
GEPA code proposer, and round-trips them through save/load.

Tools are callables/dspy.Tool objects — they are NOT serialized. Like the signature, they are
re-provided at construction; the saved ``module_src`` references them by name and resolves
against the reconstructed Flex's context at bind time.

Flex runs its code through an interpreter (no in-process mode), so tests that only inspect
``module_src`` / context use a MockInterpreter (Deno-free), while tests that bind-and-run
generated code are marked ``deno_required``.
"""

from __future__ import annotations

import re
import shutil
import textwrap

import pytest

import dspy
from dspy.predict.flex import Flex
from dspy.predict.flex.primitives_doc import PRIMITIVES_CATALOG
from dspy.primitives.code_interpreter import CodeInterpreterError
from dspy.primitives.facade import SHIM_SETUP
from dspy.teleprompt.gepa.gepa_flex_utils import CodeProposalSignature
from dspy.teleprompt.gepa.gepa_utils import DspyAdapter
from dspy.utils.dummies import DummyLM
from tests.mock_interpreter import MockInterpreter

deno_required = pytest.mark.skipif(shutil.which("deno") is None, reason="Deno is not installed")


def shout(text: str) -> str:
    """Return the text in uppercase."""
    return text.upper()


class Echo(dspy.Signature):
    """Echo the question as the answer."""

    q: str = dspy.InputField()
    a: str = dspy.OutputField()


# A tool-using module with NO predictors: forward calls the in-scope `shout` tool directly,
# so it runs LM-free (stands in for code GEPA might produce that wires a tool in).
TOOL_MODULE = textwrap.dedent("""
    class EchoModule(dspy.Module):
        def __init__(self):
            super().__init__()

        def forward(self, q):
            return dspy.Prediction(a=shout(q))
""").strip()


def test_baseline_is_rlm_with_tools() -> None:
    # With tools, the baseline delegates to dspy.RLM (which can call them via its REPL), not Predict,
    # and the tools are wired into its constructor by name.
    src = Flex(Echo, tools=[shout], interpreter_factory=lambda: MockInterpreter()).module_src
    assert "dspy.RLM(" in src
    assert "tools=[shout]" in src
    assert "dspy.Predict(" not in src


def test_reserved_bridge_tool_names_are_rejected() -> None:
    # The sandbox bridge registers its own callbacks (__dspy_construct__, __dspy_call__) alongside
    # user tools, and the shim owns `dspy`/`_dspy*` in the sandbox namespace; a user tool with a
    # reserved name would silently replace or be shadowed by them.
    for reserved in ("__dspy_call__", "_dspy_host", "dspy"):
        with pytest.raises(ValueError, match="reserved"):
            Flex(Echo, tools=[dspy.Tool(shout, name=reserved)], interpreter_factory=lambda: MockInterpreter())


def test_baseline_is_predict_without_tools() -> None:
    # Without tools, the baseline is a single dspy.Predict (no RLM, no tools arg).
    src = Flex(Echo, interpreter_factory=lambda: MockInterpreter()).module_src
    assert "dspy.Predict(" in src
    assert "dspy.RLM(" not in src
    assert "tools=[" not in src


@deno_required
def test_tool_is_in_scope_for_bound_code() -> None:
    program = Flex(Echo, tools=[shout], interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(TOOL_MODULE)
    # LM-free: forward only calls the tool (bridged from the sandbox to the host).
    assert program(q="hello").a == "HELLO"


@deno_required
def test_save_load_roundtrips_tool_using_code(tmp_path) -> None:
    path = tmp_path / "program.json"
    program = Flex(Echo, tools=[shout], interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(TOOL_MODULE)
    program.save(path)

    # Reconstruct with the SAME tools (the architecture is re-provided in code, like the
    # signature), then load the saved code — the tool reference resolves and runs.
    reloaded = Flex(Echo, tools=[shout], interpreter_factory=lambda: dspy.PythonInterpreter())
    reloaded.load(path)
    assert "shout(q)" in reloaded.module_src
    assert reloaded(q="hi").a == "HI"


@deno_required
def test_load_without_tools_cannot_resolve_them(tmp_path) -> None:
    # The baseline RLM is built with tools=[shout] in __init__; reconstructing without the
    # tool can't resolve the name in the sandbox. Loading succeeds (binding is a host-side
    # syntax check) — the failure surfaces when the code first runs (tools must be re-provided).
    path = tmp_path / "program.json"
    program = Flex(Echo, tools=[shout], interpreter_factory=lambda: dspy.PythonInterpreter())
    program.save(path)

    reloaded = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())  # tools NOT re-provided
    reloaded.load(path)
    with pytest.raises(CodeInterpreterError):
        reloaded(q="hi")


# The primitives catalog offers the proposer `dspy.Tool(func)`, so the sandbox shim has to define
# it — a name the catalog advertises but the shim lacks turns into an AttributeError at bind time,
# scoring an otherwise-fine candidate as a failure.
WRAPPED_TOOL_MODULE = textwrap.dedent("""
    class EchoModule(dspy.Module):
        def __init__(self):
            super().__init__()
            self.react = dspy.ReAct("q -> a", tools=[dspy.Tool(shout)])

        def forward(self, **inputs):
            return dspy.Prediction(a=shout(inputs["q"]))
""").strip()


@deno_required
def test_dspy_tool_wrapper_is_available_in_the_sandbox() -> None:
    program = Flex(Echo, tools=[shout], interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(WRAPPED_TOOL_MODULE)
    # forward re-runs __init__ in the sandbox, so it only succeeds if the shim defines dspy.Tool
    # and the wrapper resolves to the same host tool a bare reference would (a failed resolution
    # raises out of the ReAct's construction).
    assert program(q="hi").a == "HI"


def test_proposer_prompts_only_advertise_names_the_sandbox_defines() -> None:
    """Every `dspy.X(...)` the proposer is told to write must exist in the shim.

    The prompts and the shim are the two halves of one contract, edited in different files: a name
    advertised but not defined becomes an AttributeError at bind time, and GEPA scores the
    candidate as a failure with nothing pointing at the real cause.
    """
    # The shim only registers itself as the importable `dspy` inside the sandbox (where the host
    # bridge tools are in globals), so running it here just builds the module object to inspect.
    namespace: dict = {}
    exec(SHIM_SETUP, namespace)
    shim = namespace["dspy"]

    prompts = PRIMITIVES_CATALOG + (CodeProposalSignature.__doc__ or "")
    advertised = set(re.findall(r"dspy\.([A-Z]\w+)\(", prompts))
    assert advertised, "expected the prompts to name some primitives"
    missing = sorted(name for name in advertised if not hasattr(shim, name))
    assert not missing, f"advertised to the proposer but missing from the sandbox shim: {missing}"


def test_gepa_proposer_is_told_about_tools() -> None:
    """GEPA can optimize tool usage because the proposer is shown the available tools."""

    def _metric(gold, pred, trace=None, pred_name=None, pred_trace=None):
        return 1.0

    student = Flex(Echo, tools=[shout], interpreter_factory=lambda: MockInterpreter())
    adapter = DspyAdapter(student_module=student, metric_fn=_metric, feedback_map={})
    assert "shout" in adapter._flex_context_blurbs["self"]


# A module that AUTHORS its own helper (defined inside forward) and calls it directly — what GEPA is
# encouraged to do in a sandbox, where an authored function cannot be handed to a bridged
# sub-predictor. The helper lives in the source, so it persists with the code itself.
SELF_AUTHORED_TOOL_MODULE = textwrap.dedent("""
    class EchoModule(dspy.Module):
        def __init__(self):
            super().__init__()

        def forward(self, **inputs):
            def upcase(text: str) -> str:
                "Uppercase the text."
                return text.upper()
            return dspy.Prediction(a=upcase(inputs["q"]))
""").strip()


@deno_required
def test_self_authored_tool_persists_with_the_code(tmp_path) -> None:
    # Helpers GEPA writes itself live in module_src, so they persist exactly like the code (and like
    # instructions): reconstructing with ONLY the signature — no tools re-provided — rebuilds the
    # helper from the saved source and runs it.
    path = tmp_path / "program.json"
    flex = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())  # no user-provided tools
    flex._bind_code(SELF_AUTHORED_TOOL_MODULE)
    assert "def upcase" in flex.module_src
    flex.save(path)

    reloaded = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())  # still no tools
    reloaded.load(path)
    assert reloaded.module_src == SELF_AUTHORED_TOOL_MODULE
    assert reloaded(q="hi").a == "HI"


# Instructions baked into the generated code, on an inner predictor.
INSTR_MODULE = textwrap.dedent("""
    class EchoModule(dspy.Module):
        def __init__(self):
            super().__init__()
            self.p = dspy.Predict(dspy.Signature("q -> a", "Always answer in uppercase."))

        def forward(self, q):
            return dspy.Prediction(a=self.p(q=q).a)
""").strip()


@deno_required
def test_inner_predictor_instructions_persist(tmp_path) -> None:
    # The instructions GEPA writes into the code persist through save/load via module_src alone:
    # each forward rebuilds the predictor from the source, instructions included — observable in
    # the prompt the LM actually receives.
    path = tmp_path / "program.json"
    lm = DummyLM([{"a": "HI"}] * 4)
    dspy.configure(lm=lm)
    flex = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    flex._bind_code(INSTR_MODULE)
    flex(q="hi")
    assert "Always answer in uppercase." in str(lm.history[-1]["messages"])
    flex.save(path)

    reloaded = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    reloaded.load(path)
    reloaded(q="hi")
    assert "Always answer in uppercase." in str(lm.history[-1]["messages"])


def double(n: int) -> str:
    """Double the number."""
    return str(n * 2)


@deno_required
def test_direct_tool_calls_go_through_the_tool_wrapper() -> None:
    # Tools keep their dspy.Tool wrapper when the sandbox calls them directly, so DSPy's argument
    # validation applies: n=3 works, n="3" is rejected loudly — the raw underlying function would
    # have silently returned "33".
    def src(arg: str) -> str:
        return textwrap.dedent(
            f"""
            class EchoModule(dspy.Module):
                def __init__(self):
                    super().__init__()

                def forward(self, q):
                    return dspy.Prediction(a=double(n={arg}))
            """
        ).strip()

    program = Flex(Echo, tools=[dspy.Tool(double)], interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(src("3"))
    assert program(q="x").a == "6"
    program._bind_code(src('"3"'))
    with pytest.raises(CodeInterpreterError, match="not of type 'integer'"):
        program(q="x")


@deno_required
def test_direct_tool_calls_receive_restored_custom_type_inputs() -> None:
    """A custom-type input (dspy.Image) crosses into the sandbox as its serialized string; a
    direct tool call must hand the tool the original object, exactly as a bridged predictor
    receives it via ``_Invocation.call``."""
    seen: dict = {}

    def describe(img) -> str:
        """Describe an image."""
        seen["img"] = img
        return "a cat"

    class Caption(dspy.Signature):
        image: dspy.Image = dspy.InputField()
        a: str = dspy.OutputField()

    src = textwrap.dedent(
        """
        class CaptionModule(dspy.Module):
            def __init__(self):
                super().__init__()

            def forward(self, **inputs):
                return dspy.Prediction(a=describe(img=inputs["image"]))
        """
    ).strip()

    program = Flex(Caption, tools=[describe], interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(src)
    image = dspy.Image(url="https://example.com/x.png")
    assert program(image=image).a == "a cat"
    assert seen["img"] is image  # the original object, not the serialized tag string