Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
11.7 KB
Raw Blame
import json
from datetime import date
from typing import Literal

import pytest
from PIL import Image
from pydantic import BaseModel, ValidationError

import dspy
from dspy.adapters.decision import record_evidence
from dspy.experimental import Score
from dspy.utils.dummies import DummyLM

EDGES = {"top": 0.13, "bottom": 0.81, "left": 0.27, "right": 0.64}


class Locate(dspy.Signature):
    """Find the target in the photo."""

    photo: dspy.Image = dspy.InputField()
    box: dspy.BoundingBox = dspy.OutputField(desc="The dog in inputs.photo.")


def test_geometry_round_trip_and_boundaries():
    box = dspy.BoundingBox(**EDGES)
    assert isinstance(box, dspy.Type)
    assert box.model_dump() == EDGES
    assert dspy.BoundingBox.model_validate_json(box.model_dump_json()) == box
    assert json.loads(box.format()) == EDGES
    assert dspy.BoundingBox(top=0, bottom=1, left=0, right=1).right == 1
    with pytest.raises(ValidationError):
        box.left = 0.9


@pytest.mark.parametrize("change", [
    {"top": -0.01}, {"right": 1.01}, {"left": float("nan")}, {"bottom": float("inf")},
    {"top": True}, {"left": "0.27"}, {"top": 0.81}, {"left": 0.65}, {"confidence": 0.9},
])
def test_invalid_geometry_is_not_silently_repaired(change):
    with pytest.raises(ValidationError):
        dspy.BoundingBox(**{**EDGES, **change})


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("adapter_class", [dspy.ChatAdapter, dspy.JSONAdapter, dspy.XMLAdapter, dspy.TwoStepAdapter])
async def test_boxes_are_structured_outputs_inputs_and_saved_demos(adapter_class, asynchronous, tmp_path):
    class Regions(BaseModel):
        boxes: list[dspy.BoundingBox]

    signature = dspy.Signature({
        "previous": (dspy.BoundingBox, dspy.InputField()),
        "region": (Regions, dspy.OutputField()),
        "missing": (dspy.BoundingBox | None, dspy.OutputField()),
    })
    predict = dspy.Predict(signature)
    box = dspy.BoundingBox(**EDGES)
    predict.demos = [dspy.Example(previous=box, region=Regions(boxes=[box]), missing=None)]
    predict.save(tmp_path / "boxes.json")
    predict.load(tmp_path / "boxes.json")
    assert predict.demos[0]["previous"] == EDGES
    assert predict.demos[0]["region"] == {"boxes": [EDGES]}
    answer = {"region": {"boxes": [EDGES]}, "missing": None}
    if adapter_class is dspy.TwoStepAdapter:
        adapter = adapter_class(extraction_model=DummyLM([answer]))
        lm = DummyLM([{"answer": "The region contains one box: top 0.13, bottom 0.81, left 0.27, right 0.64. "
                                  "The missing box is null."}])
    else:
        adapter = adapter_class()
        lm = DummyLM([answer], adapter=adapter)
    with dspy.context(adapter=adapter):
        result = await predict.acall(previous=box, lm=lm) if asynchronous else predict(previous=box, lm=lm)
    assert result.region.boxes == [box]
    assert result.missing is None
    assert "CUSTOM-TYPE" not in json.dumps(lm.history[-1]["messages"])


@pytest.mark.parametrize("left", ["0.27", "true"])
def test_predict_boxes_from_literal_nested_xml(left, monkeypatch):
    from tests.clients.test_lm_engine_execution import native_transport, wire

    class Regions(BaseModel):
        boxes: list[dspy.BoundingBox]
        label: str

    signature = dspy.Signature({"region": (Regions, dspy.OutputField())})
    completion = (
        f"<region><boxes><item><top>0.13</top><bottom>0.81</bottom><left>{left}</left>"
        "<right>0.64</right></item></boxes><label>0.27</label></region>"
    )
    native_transport(monkeypatch, [wire(completion), wire(completion)])
    lm = dspy.LM("openai/gpt-4o-mini", engine="lm15", cache=False)
    try:
        with dspy.context(adapter=dspy.XMLAdapter()):
            if left == "true":
                with pytest.raises(dspy.AdapterParseError):
                    dspy.Predict(signature)(lm=lm)
            else:
                result = dspy.Predict(signature)(lm=lm)
                assert result.region.boxes == [dspy.BoundingBox(**EDGES)]
                assert result.region.label == "0.27"
    finally:
        lm.close()


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("mixed", [False, True])
async def test_predict_native_boxes_evidence_images_and_cache(asynchronous, mixed, monkeypatch):
    from dspy import lm15
    from dspy._vendor.lm15.testing import FakeTransport
    from tests.clients.test_lm_engine_execution import DualResponse

    signature = Locate
    answers = {"box": {"type": "box", "box": EDGES, "confidence": 0.83}}
    if mixed:
        signature = signature.append("second", dspy.OutputField(desc="The other target."), type_=dspy.BoundingBox)
        signature = signature.append("visible", dspy.OutputField(desc="Is a dog visible?"), type_=bool)
        signature = signature.append("kind", dspy.OutputField(desc="What kind of animal?"), type_=Literal["dog", "cat"])
        signature = signature.append("size", dspy.OutputField(desc="Size, small to large."), type_=Score["small", "medium", "large"])
        answers.update({
            "second": {"type": "box", "box": {"top": 0.21, "bottom": 0.67, "left": 0.04, "right": 0.19}, "confidence": 0.62},
            "visible": {"type": "noul", "noul": 0.91},
            "kind": {"type": "choice", "choice": "dog", "probabilities": {"dog": 0.8, "cat": 0.2}, "confidence": 0.7},
            "size": {"type": "score", "score": 1.6, "probabilities": {"0": 0.1, "1": 0.2, "2": 0.7}, "confidence": 0.6},
        })
    wire_response = {"model": "jev-1.14.0", "answers": answers, "usage": {"input_tokens": 31, "output_tokens": 8}}
    transport = FakeTransport([DualResponse(200, json.dumps(wire_response).encode())])
    monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
    lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", num_retries=0)
    image = dspy.Image(Image.new("RGB", (320, 280), "red"))
    predict = dspy.Predict(signature)
    predict.demos = [dspy.Example(photo=image, box=dspy.BoundingBox(**EDGES))]
    try:
        with dspy.context(adapter=None), record_evidence() as evidence:
            for _ in range(2):
                result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)
                assert result.box.model_dump() == EDGES
                response = lm.history[-1]["response"]
                assert response.provider_data["typesafe"]["answers"] == answers
                assert response.data["box"] == EDGES
                if mixed:
                    assert result.second.model_dump() == answers["second"]["box"]
                    assert result.visible is True and result.kind == "dog" and result.size.level == 2
                    assert result.size.value == pytest.approx(1.6)
                    assert set(response.probabilities) == {"visible", "kind", "size"}
                    assert response.probabilities["visible"]["true"] == 0.91
                else:
                    assert response.probabilities is None
            if mixed:
                predict.fields = {"visible": {"threshold": 0.95}, "kind": {"weights": {"cat": 10}},
                                  "size": {"cuts": [0.4, 1.8]}}
                result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)
                assert result.visible is False and result.kind == "cat" and result.size.level == 1
                assert result.box.model_dump() == EDGES
                predict.fields["box"] = {"threshold": 0.9}
                with pytest.raises(ValueError, match="unsupported outputs"):
                    if asynchronous:
                        await predict.acall(photo=image, lm=lm)
                    else:
                        predict(photo=image, lm=lm)
            assert next(answer for _, name, answer in evidence if name == "box") == answers["box"]
        assert len(transport.requests) == 1
        request = transport.requests[0]
        payload = json.loads(request.body)
        assert payload["questions"]["box"] == {"type": "box", "instructions": "The dog in inputs.photo."}
        assert len(payload["assets"]) == 2  # input and demonstration both stay images
        assert all(asset["data_base64"] == image.url.split(",", 1)[1] for asset in payload["assets"].values())
        assert "Find the target" in str(payload["state"]) and "demos" in str(payload["state"])
        assert dict(request.headers)["TypeSafe-Danger-Zone-Images"] == "true"
        assert lm.history[-1]["usage"] == {}
    finally:
        await lm.aclose()


@pytest.mark.parametrize("with_image", [False, True])
def test_native_state_serializes_inputs_and_demos(with_image, monkeypatch):
    from dspy import lm15
    from dspy._vendor.lm15.testing import FakeTransport
    from tests.clients.test_lm_engine_execution import DualResponse

    signature = dspy.Signature({"metadata": (dict, dspy.InputField()), "box": (dspy.BoundingBox, dspy.OutputField())})
    inputs = {"metadata": {"point": (9, 2), "date": date(2026, 10, 19), "tags": {"input"}}}
    demo = {"metadata": {"point": (4, 7), "date": date(2025, 3, 8), "tags": {"demo"}}, "box": dspy.BoundingBox(**EDGES)}
    if with_image:
        signature = signature.prepend("photo", dspy.InputField(), type_=dspy.Image)
        inputs["photo"] = demo["photo"] = dspy.Image(Image.new("RGB", (320, 280), "red"))
    response = {"model": "jev-1.14.0", "answers": {"box": {"type": "box", "box": EDGES, "confidence": 0.8}}}
    transport = FakeTransport([DualResponse(200, json.dumps(response).encode())])
    monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
    lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)
    predict = dspy.Predict(signature)
    predict.demos = [dspy.Example(**demo)]
    try:
        assert predict(lm=lm, **inputs).box.model_dump() == EDGES
        payload = json.loads(transport.requests[0].body)
        if with_image:
            assert len(payload["assets"]) == 2
        else:
            assert payload["state"]["inputs"]["metadata"] == {"point": [9, 2], "date": "2026-10-19", "tags": ["input"]}
            assert payload["state"]["demos"][0]["metadata"] == {"point": [4, 7], "date": "2025-03-08", "tags": ["demo"]}
    finally:
        lm.close()


@pytest.mark.parametrize("annotation", [list[dspy.BoundingBox], dspy.BoundingBox | None, str])
def test_unsupported_native_outputs_are_refused(annotation, monkeypatch):
    from dspy import lm15
    from dspy._vendor.lm15.testing import FakeTransport

    transport = FakeTransport([])
    monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
    signature = Locate.with_updated_fields("box", type_=annotation)
    lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)
    try:
        with dspy.context(adapter=None), pytest.raises(ValueError, match="Unsupported System One output"):
            dspy.Predict(signature)(photo=dspy.Image(Image.new("RGB", (320, 280))), lm=lm)
        assert not transport.requests
    finally:
        lm.close()


def test_adapter_rejects_invalid_generated_geometry():
    with pytest.raises(dspy.AdapterParseError, match="top < bottom"):
        dspy.JSONAdapter().parse(Locate, json.dumps({"box": {**EDGES, "top": 0.9}}))

    from tests.predict.test_decision_parameters import FakeClient

    signature = dspy.Signature({"box": (dspy.BoundingBox, dspy.OutputField())})
    lm = FakeClient(answers={"box": {"box": {**EDGES, "top": 0.9}, "confidence": 0.8}})
    with pytest.raises(dspy.AdapterParseError, match="top < bottom"):
        dspy.Predict(signature)(lm=lm)