Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
13.4 KB
Raw Blame
"""dspy.Flex binds a module's source (``module_src``) and makes it usable like any dspy.Module.

Binding a source — the deterministic baseline at construction, or a GEPA-optimized decomposition —
records it host-side (a syntax check, no interpreter); each forward runs it in a fresh sandbox,
constructing its predictors into that forward's own bridge invocation (nothing is attached to the
module, so parallel forwards can't overwrite each other's predictors). Everything
round-trips through Module.save / load so the generated code can be persisted and rerun. Tests
that run predictors need Deno; pure ``module_src`` checks use a MockInterpreter instead.
(The baseline's RLM-vs-Predict shape is covered in test_tools.py; GEPA's code rewriting in
test_flex_gepa.py.)
"""

from __future__ import annotations

import shutil
import textwrap

import pytest

import dspy
from dspy.predict.flex.flex import Flex
from dspy.utils.dummies import DummyLM
from tests.mock_interpreter import MockInterpreter

deno_required = pytest.mark.skipif(shutil.which("deno") is None, reason="Deno is not installed")

# A plain dspy.Predict module we bind and run with a DummyLM, standing in for a GEPA-optimized
# decomposition — end-to-end forward tests bind this instead of the construction baseline.
ECHO_MODULE = textwrap.dedent("""
    class EchoModule(dspy.Module):
        def __init__(self):
            super().__init__()
            self.echo = dspy.Predict("q -> a")

        def forward(self, q):
            out = self.echo(q=q)
            return dspy.Prediction(a=out.a)
""").strip()


class Echo(dspy.Signature):
    """Echo the question as the answer."""

    q: str = dspy.InputField()
    a: str = dspy.OutputField()


@deno_required
def test_forward_attaches_nothing_to_the_module() -> None:
    # A forward's predictors live in that forward's bridge invocation, never on the shared Flex:
    # parallel forwards (e.g. Evaluate(num_threads>1)) run on one module object, and attached
    # predictors would let one forward overwrite another's. The module's surface stays clean,
    # and named_predictors() reports no tunable units — the Flex's update unit is its module_src.
    dspy.configure(lm=DummyLM([{"a": "hi"}]))
    program = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    program(q="hello")
    assert not hasattr(program, "predict")
    assert "predict" not in [n for n, _ in program.named_parameters()]
    assert program.named_predictors() == []


def test_construction_writes_nothing_to_disk(tmp_path, monkeypatch) -> None:
    # The implementation lives in memory; nothing is written until you call save(). Only module_src is
    # inspected here, so a MockInterpreter keeps it Deno-free.
    monkeypatch.chdir(tmp_path)
    program = Flex(Echo, interpreter_factory=lambda: MockInterpreter())
    assert program.module_src is not None
    assert "dspy.Predict(" in program.module_src
    # Flex itself persists nothing — no module file written until save() is called.
    # (dspy's global `.dspy_cache` may appear; that's unrelated to Flex persistence.)
    assert not list(tmp_path.glob("*.py"))
    assert not list(tmp_path.glob("*.json"))


@deno_required
def test_save_load_roundtrips_generated_code(tmp_path) -> None:
    """Module.save/load carries the generated code (module_src), like instruction state."""
    path = tmp_path / "program.json"
    program = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(ECHO_MODULE)  # stand in for a GEPA-optimized decomposition
    program.save(path)

    # A fresh Flex starts on the Predict baseline...
    reloaded = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    assert "self.echo" not in reloaded.module_src
    # ...and load() rebinds the saved code (no interpreter or LM needed: the code runs, and
    # constructs its predictors, at each forward).
    reloaded.load(path)
    assert "self.echo" in reloaded.module_src
    assert not hasattr(reloaded, "echo")


@deno_required
def test_nested_flex_state_roundtrips_inside_a_program(tmp_path) -> None:
    """A Flex nested in a larger program persists like a top-level one.

    The parent's state must carry the sub-module's `module_src`, not just its predictors' state:
    those predictors are *created by* that source, so without it a reload has nothing to attach
    them to and the optimized code is lost.
    """
    path = tmp_path / "program.json"

    class Program(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.sibling = dspy.Predict("x -> y")
            self.flex = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    program = Program()
    program.flex._bind_code(ECHO_MODULE)  # stand in for a GEPA-optimized decomposition
    program.sibling.signature = program.sibling.signature.with_instructions("tuned sibling")

    state = program.dump_state()
    assert state["flex"]["module_src"] == ECHO_MODULE  # the code, not just its predictors
    assert "sibling" in state  # a plain predictor stays a flat top-level entry
    # The Flex's internals belong to its own state; the parent must not dump them separately.
    assert not any(key.startswith("flex.") for key in state)

    program.save(path)

    reloaded = Program()
    assert "self.echo" not in reloaded.flex.module_src  # a fresh program is on the baseline
    reloaded.load(path)

    assert reloaded.flex.module_src == ECHO_MODULE
    assert reloaded.sibling.signature.instructions == "tuned sibling"


def test_flex_is_a_parameter_leaf_in_parent_programs() -> None:
    # Saving, GEPA discovery, and reset_copy all walk named_parameters(); a Flex appears there
    # as one state-bearing leaf (like dspy.Predict), so nothing inside it is enumerated — or
    # optimized — behind the code's back.
    class Program(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.sibling = dspy.Predict("x -> y")
            self.flex = Flex(Echo, interpreter_factory=lambda: MockInterpreter())

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    program = Program()
    # Even a predictor set directly on the Flex (the bridge itself never attaches any) stays
    # behind the leaf.
    program.flex.echo = dspy.Predict("q -> a")

    params = dict(program.named_parameters())
    assert isinstance(params["flex"], Flex)
    assert not any(name.startswith("flex.") for name in params)
    # The internals are not tunable units from any vantage point; the parent sees one leaf.
    assert program.flex.named_predictors() == []
    assert [n for n, _ in program.named_predictors()] == ["sibling"]


def test_parent_set_lm_and_get_lm_are_symmetric() -> None:
    # set_lm writes to every LM-bearing leaf (Predicts and Flexes); get_lm reads the same set —
    # so set-then-get is an identity, a flex-only program has a unique LM rather than an error,
    # and a divergent flex LM is reported instead of silently ignored.
    class Program(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.sibling = dspy.Predict("x -> y")
            self.flex = Flex(Echo, interpreter_factory=MockInterpreter)

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    program = Program()
    lm = dspy.LM("openai/gpt-4o-mini")
    program.set_lm(lm)
    assert program.sibling.lm is lm
    assert program.flex.get_lm() is lm
    assert program.get_lm() is lm  # read-your-writes

    program.flex.lm = dspy.LM("openai/gpt-4.1")
    with pytest.raises(ValueError, match="Multiple LMs"):
        program.get_lm()

    class OnlyFlex(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.flex = Flex(Echo, interpreter_factory=MockInterpreter)

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    only = OnlyFlex()
    only.set_lm(lm)
    assert only.get_lm() is lm  # previously raised "Multiple LMs" on the empty predictor list


def test_set_lm_is_stored_and_survives_rebind_and_save(tmp_path) -> None:
    # set_lm stores one module-level LM (one home), applied via context at forward time — so it
    # survives rebinds and save/load, unlike writing .lm onto per-forward internals that the
    # next forward rebuilds from scratch.
    flex = Flex(Echo, interpreter_factory=MockInterpreter)
    lm = dspy.LM("openai/gpt-4o-mini")
    flex.set_lm(lm)
    assert flex.get_lm() is lm

    flex._bind_code(ECHO_MODULE)  # a rebind must not lose the module-level LM
    assert flex.get_lm() is lm

    path = tmp_path / "flex.json"
    flex.save(path)
    reloaded = Flex(Echo, interpreter_factory=MockInterpreter)
    reloaded.load(path)
    assert reloaded.get_lm() is not None
    assert reloaded.get_lm().model == lm.model

    # Like Predict: loading a state without an LM clears any previously set one.
    reloaded.load_state({"module_src": ECHO_MODULE, "lm": None})
    assert reloaded.get_lm() is None

    flex.set_lm(lm)
    assert flex.reset_copy().get_lm() is None  # reset clears the LM, like Predict.reset


@deno_required
def test_context_lm_reaches_bridged_calls() -> None:
    # A Flex holds no LM of its own (its state is only module_src); like any module, its
    # predictor calls resolve the ambient LM — including a caller's dspy.context, since the
    # bridged calls run host-side in the calling thread.
    program = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(ECHO_MODULE)
    with dspy.context(lm=DummyLM([{"a": "from-context-lm"}])):
        assert program(q="hello").a == "from-context-lm"


def test_reset_copy_keeps_optimized_code_and_clears_the_lm() -> None:
    # reset_copy() calls .reset() on every parameter leaf. Predict.reset() clears demos/lm but
    # keeps tuned instructions; a Flex resets the same way — the LM is cleared, the optimized
    # module_src is kept — so optimizer chains (e.g. GEPA then BootstrapFewShot, which compiles
    # a reset copy) don't silently discard the discovered code. There is no other internal state:
    # predictors are rebuilt from the source inside each forward's bridge invocation.
    class Program(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.flex = Flex(Echo, interpreter_factory=lambda: MockInterpreter())

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    program = Program()
    program.flex._bind_code(ECHO_MODULE)  # stand in for a GEPA-optimized decomposition
    program.flex.set_lm(dspy.LM("openai/gpt-4o-mini"))

    fresh = program.reset_copy()
    assert fresh.flex.module_src == ECHO_MODULE  # the optimized code survives, like instructions
    assert fresh.flex.get_lm() is None  # the LM is cleared, like Predict.reset
    assert program.flex.get_lm() is not None  # the original is untouched


def test_state_layout_unchanged_without_a_flex() -> None:
    # Programs with no Flex keep the flat `{param_path: state}` layout, so existing saved state
    # still loads.
    class Plain(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.first = dspy.Predict("x -> y")
            self.second = dspy.Predict("y -> z")

        def forward(self, **kwargs):
            return self.second(**self.first(**kwargs))

    assert sorted(Plain().dump_state()) == ["first", "second"]


def test_save_program_roundtrips_via_cloudpickle(tmp_path) -> None:
    # save_program=True cloudpickles the live program object. The bridge (sessions, locks,
    # thread-locals) is excluded from the pickle and rebuilt session-less on load.
    class Program(dspy.Module):
        def __init__(self) -> None:
            super().__init__()
            self.sibling = dspy.Predict("x -> y")
            self.flex = Flex(Echo, interpreter_factory=MockInterpreter)

        def forward(self, **kwargs):
            return self.flex(**kwargs)

    program = Program()
    program.flex._bind_code(ECHO_MODULE)
    assert "_bridge" not in program.flex.__getstate__()
    program.save(tmp_path, save_program=True)

    loaded = dspy.load(str(tmp_path), allow_pickle=True)
    assert loaded.flex.module_src == ECHO_MODULE
    assert loaded.flex._bridge._flex is loaded.flex
    assert loaded.flex._bridge._module_src == ECHO_MODULE


@deno_required
def test_save_program_end_to_end_forward(tmp_path) -> None:
    """save_program=True with a live Deno session: reload elsewhere and run it."""
    program = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(ECHO_MODULE)
    program.save(tmp_path, save_program=True)

    dspy.configure(lm=DummyLM([{"a": "echoed-back"}]))
    loaded = dspy.load(str(tmp_path), allow_pickle=True)
    assert loaded.module_src == ECHO_MODULE
    result = loaded(q="hello")
    assert result.a == "echoed-back"


@deno_required
def test_end_to_end_forward_after_save_load(tmp_path) -> None:
    """End-to-end: bind a plain dspy.Predict class, save it, reload into a fresh Flex, run it."""
    path = tmp_path / "program.json"
    program = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    program._bind_code(ECHO_MODULE)
    program.save(path)

    dspy.configure(lm=DummyLM([{"a": "echoed-back"}]))
    reloaded = Flex(Echo, interpreter_factory=lambda: dspy.PythonInterpreter())
    reloaded.load(path)
    assert "self.echo" in reloaded.module_src
    result = reloaded(q="hello")
    assert isinstance(result, dspy.Prediction)
    assert result.a == "echoed-back"