Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add BoundingBox type with automatic Jev image support (#10665)

Isaac Miller6ee6ca3d427a
diff --git a/docs/docs/api/primitives/BoundingBox.md b/docs/docs/api/primitives/BoundingBox.md
new file mode 100644
index 000000000..1dcef007c
--- /dev/null
+++ b/docs/docs/api/primitives/BoundingBox.md
@@ -0,0 +1,74 @@
+# dspy.BoundingBox
+
+`BoundingBox` is a `dspy.Type` for an axis-aligned rectangle. Its four required
+fields are `top`, `bottom`, `left`, and `right`: finite numbers in [0, 1], measured
+from the image's top-left corner. `top < bottom` and `left < right` are required.
+Invalid geometry raises a validation error; DSPy does not clamp or reorder it.
+Boxes are immutable, serialize as JSON objects, and can be input or output fields.
+
+## Locate an object with Jev
+
+Set `TYPESAFE_API_KEY`, then pass an image to a signature with a `BoundingBox` output:
+
+```python
+import dspy
+
+class Locate(dspy.Signature):
+ photo: dspy.Image = dspy.InputField()
+ visible: bool = dspy.OutputField(desc="Is there a coffee mug in inputs.photo?")
+ mug: dspy.BoundingBox = dspy.OutputField(desc="The coffee mug in inputs.photo.")
+
+lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15")
+predict = dspy.Predict(Locate)
+with dspy.context(lm=lm):
+ result = predict(photo=dspy.Image.from_path("kitchen.jpg"))
+
+if result.visible:
+ print(result.mug.model_dump()) # top, bottom, left, right
+```
+
+`Predict` selects the native Jev route automatically; no separate adapter is
+needed. For async calls, use `await predict.acall(...)` in the same context.
+There is **no not-found box**: ask a presence question and ignore the box when
+the target is absent. Invalid native geometry raises `dspy.AdapterParseError`.
+
+To convert to pixels, multiply `left` and `right` by the image width, and `top`
+and `bottom` by its height. Use the dimensions of the image sent to the model;
+map cropped-image coordinates back to the original yourself.
+
+### Jev requirements and limits
+
+- Images and boxes are experimental and require Jev 1.14 or `jev-preview`.
+ DSPy sends the required experimental-image header automatically.
+- Use `Predict` with one top-level box field per target. Native Jev does not
+ support nested, list, or nullable boxes, generated text, tools, or streaming.
+- Boxes can accompany `bool`, `Literal`, and experimental `Noul`, `Choice`, or
+ `Score` outputs. Decision calibration applies to those fields, not boxes;
+ use `Score` for ordinal scoring, not an integer `Literal`.
+- Load PNG/JPEG bytes with `Image.from_path` or `Image.from_url`; bare URLs are
+ not fetched. DSPy does not resize or apply EXIF orientation here. See the
+ [image limits](https://docs.typesafe.ai/images).
+- For multiple images, name the target and its input image in the output field
+ description. Images in inputs and demonstrations retain their named context.
+
+Provider confidence stays in the LM response, separate from geometry; boxes
+have no entry in `response.probabilities`. Neither valid geometry nor high
+confidence guarantees accurate localization. See TypeSafe's
+[Box guide](https://docs.typesafe.ai/primitives/box) for reference images,
+confidence thresholds, and tiling.
+
+## Use with generative LMs
+
+The same type works with `ChatAdapter`, `JSONAdapter`, `XMLAdapter`, and
+`TwoStepAdapter`, including nested models, `list[BoundingBox]`, and optional
+boxes, subject to provider support. Generated coordinates receive the same
+validation but have no native TypeSafe confidence.
+
+<!-- START_API_REF -->
+::: dspy.BoundingBox
+ handler: python
+ options:
+ show_root_heading: true
+ heading_level: 2
+ show_source: true
+<!-- END_API_REF -->
diff --git a/docs/docs/api/primitives/index.md b/docs/docs/api/primitives/index.md
index c7102ccc8..5438d292e 100644
--- a/docs/docs/api/primitives/index.md
+++ b/docs/docs/api/primitives/index.md
@@ -4,6 +4,7 @@ API reference for DSPy primitives. Select a page below.
<!-- START_API_INDEX -->
- [Audio](Audio.md)
+- [BoundingBox](BoundingBox.md)
- [Code](Code.md)
- [Example](Example.md)
- [History](History.md)
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 23aa42d28..edcb74585 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -156,6 +156,7 @@ nav:
- Primitives:
- api/primitives/index.md
- Audio: api/primitives/Audio.md
+ - BoundingBox: api/primitives/BoundingBox.md
- Code: api/primitives/Code.md
- Example: api/primitives/Example.md
- History: api/primitives/History.md
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 1813f9c2b..d96c46b9d 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -10,7 +10,7 @@ from dspy.predict.flex import Flex
from dspy.evaluate import Evaluate # isort: skip
from dspy.clients import * # isort: skip
-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip
+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, BoundingBox, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip
from dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skip
from dspy.utils.exceptions import (
AdapterParseError,
diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py
index 74e199f18..70688769c 100644
--- a/dspy/adapters/__init__.py
+++ b/dspy/adapters/__init__.py
@@ -2,7 +2,19 @@ from dspy.adapters.base import Adapter
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.json_adapter import JSONAdapter
from dspy.adapters.two_step_adapter import TwoStepAdapter
-from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCallResults, ToolCalls, Type
+from dspy.adapters.types import (
+ Audio,
+ BoundingBox,
+ Code,
+ File,
+ History,
+ Image,
+ Reasoning,
+ Tool,
+ ToolCallResults,
+ ToolCalls,
+ Type,
+)
from dspy.adapters.xml_adapter import XMLAdapter
__all__ = [
@@ -21,4 +33,5 @@ __all__ = [
"ToolCalls",
"ToolCallResults",
"Reasoning",
+ "BoundingBox",
]
diff --git a/dspy/adapters/decision.py b/dspy/adapters/decision.py
index b1af846b2..9e25772b4 100644
--- a/dspy/adapters/decision.py
+++ b/dspy/adapters/decision.py
@@ -7,12 +7,15 @@ import warnings
from contextlib import contextmanager
from functools import lru_cache
-from pydantic import BaseModel, ConfigDict, Field, create_model
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, create_model
from dspy.adapters.decision_state import DecisionState
+from dspy.adapters.types.base_type import split_message_content_for_custom_types
+from dspy.adapters.types.bounding_box import BoundingBox
from dspy.adapters.types.decision import Choice, Noul, Probability, Score
from dspy.dsp.utils.settings import settings
-from dspy.lm15 import Config, DataPart, Message, Request, choice, score, yes_no
+from dspy.lm15 import Config, DataPart, Message, Request, bounding_box, choice, request_from_openai_chat, score, yes_no
+from dspy.utils.exceptions import AdapterParseError
@contextmanager
@@ -63,7 +66,7 @@ def resolve_adapter(lm, adapter, signature, fields, declared_signature=None):
UserWarning,
stacklevel=2,
)
- return DecisionAdapter(adapter, state, system_one) if state.types else adapter
+ return DecisionAdapter(adapter, state, system_one) if system_one or state.types else adapter
class DecisionAdapter:
@@ -75,7 +78,7 @@ class DecisionAdapter:
self.system_one = system_one
def _prepare(self, lm, signature, demos, inputs, lm_kwargs):
- from dspy.adapters.utils import get_field_description_string
+ from dspy.adapters.utils import get_field_description_string, serialize_for_json
from dspy.predict.predict import serialize_object
if settings.send_stream is not None:
@@ -90,12 +93,11 @@ class DecisionAdapter:
state = {
"instructions": signature.instructions,
"input_fields": get_field_description_string(signature.input_fields),
- "inputs": serialize_object({k: v for k, v in inputs.items() if k in signature.input_fields}),
+ "inputs": {k: v for k, v in inputs.items() if k in signature.input_fields},
}
if demos:
- state["demos"] = [
- serialize_object({k: v for k, v in demo.items() if k in signature.fields}) for demo in demos
- ]
+ state["demos"] = [{k: v for k, v in demo.items() if k in signature.fields} for demo in demos]
+ state = serialize_for_json(state)
properties = {}
for name, kind in types.items():
if issubclass(kind, Noul):
@@ -110,10 +112,24 @@ class DecisionAdapter:
"use a nonempty label such as 'no_match'."
)
properties[name] = choice(name, list(criteria))
+ for name, field in signature.output_fields.items():
+ if field.annotation is BoundingBox:
+ desc = field.json_schema_extra.get("desc", "")
+ instructions = name if desc == f"${{{name}}}" else desc or name
+ properties[name] = bounding_box(instructions)
+ questions[name] = {"type": "box", "instructions": instructions}
+ # Keep text-only state structured. For media, reuse DSPy's marker
+ # expansion and lm15's conversion rather than encoding images here.
+ messages = split_message_content_for_custom_types([
+ {"role": "user", "content": json.dumps(state, ensure_ascii=False)}
+ ])
+ message = Message.user(DataPart(value=state))
+ if isinstance(messages[0]["content"], list):
+ message = request_from_openai_chat({"model": lm.model, "messages": messages}).messages[0]
# The schema declares answer spaces; native questions retain structured instructions and criteria.
response_format = {"type": "json_schema", "schema": {"type": "object", "properties": properties}}
return Request(
- model=lm.model, messages=(Message.user(DataPart(value=state)),),
+ model=lm.model, messages=(message,),
config=Config(response_format=response_format, extensions={"questions": questions}),
)
for name, kind in types.items():
@@ -134,6 +150,18 @@ class DecisionAdapter:
results = []
for completion in completions:
answers = {}
+ boxes = {}
+ if self.system_one:
+ for name, field in self.state.signature.output_fields.items():
+ if field.annotation is BoundingBox:
+ answers[name] = copy.deepcopy(completion[name])
+ try:
+ boxes[name] = BoundingBox.model_validate(answers[name]["box"])
+ except ValidationError as error:
+ raise AdapterParseError(
+ adapter_name="DecisionAdapter", signature=self.state.signature,
+ lm_response=json.dumps(completion), message=str(error),
+ ) from error
for name, kind in types.items():
answer = completion[name]
if isinstance(answer, BaseModel):
@@ -152,7 +180,7 @@ class DecisionAdapter:
caller = (settings.caller_modules or [None])[-1]
with lock:
log.extend((caller, name, copy.deepcopy(answer)) for name, answer in answers.items())
- results.append({**completion, **self.state._decode(answers)})
+ results.append({**completion, **boxes, **self.state._decode(answers)})
return results
def __call__(self, lm, lm_kwargs, signature, demos, inputs):
diff --git a/dspy/adapters/decision_state.py b/dspy/adapters/decision_state.py
index 198567840..dc7f746f0 100644
--- a/dspy/adapters/decision_state.py
+++ b/dspy/adapters/decision_state.py
@@ -8,6 +8,7 @@ from typing import get_args
from pydantic import JsonValue, TypeAdapter
+from dspy.adapters.types.bounding_box import BoundingBox
from dspy.adapters.types.decision import Choice, Noul, Score, decision_type
_JSON_ADAPTER = TypeAdapter(JsonValue)
@@ -27,6 +28,9 @@ class DecisionState:
self.types = {}
self.fields = {}
for name, field in signature.output_fields.items():
+ # Native boxes carry geometry, not calibratable decision values.
+ if system_one and field.annotation is BoundingBox:
+ continue
if not system_one and name not in fields and not any(
isinstance(annotation, type) and issubclass(annotation, (Noul, Score, Choice))
for annotation in (field.annotation, *field.metadata)
diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.py
index 36403e74d..9ea5024a4 100644
--- a/dspy/adapters/types/__init__.py
+++ b/dspy/adapters/types/__init__.py
@@ -1,5 +1,6 @@
from dspy.adapters.types.audio import Audio
from dspy.adapters.types.base_type import Type
+from dspy.adapters.types.bounding_box import BoundingBox
from dspy.adapters.types.code import Code
from dspy.adapters.types.file import File
from dspy.adapters.types.history import History
@@ -7,4 +8,4 @@ from dspy.adapters.types.image import Image
from dspy.adapters.types.reasoning import Reasoning
from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls
-__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "ToolCallResults", "Code", "Reasoning"]
+__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "ToolCallResults", "Code", "Reasoning", "BoundingBox"]
diff --git a/dspy/adapters/types/bounding_box.py b/dspy/adapters/types/bounding_box.py
new file mode 100644
index 000000000..73f0520ba
--- /dev/null
+++ b/dspy/adapters/types/bounding_box.py
@@ -0,0 +1,49 @@
+import json
+
+from pydantic import ConfigDict, Field, field_validator, model_serializer, model_validator
+
+from dspy.adapters.types.base_type import Type
+
+
+class BoundingBox(Type):
+ """An axis-aligned rectangle in normalized image coordinates.
+
+ Edges are finite numbers in [0, 1], measured from the top-left corner.
+ Multiply left/right by the image width and top/bottom by its height to
+ obtain pixel coordinates. Use the dimensions of the upright image actually
+ sent to the model, not those of an original image before resizing/cropping.
+
+ Boxes have positive area: top < bottom and left < right. Invalid outputs
+ are rejected, never reordered or clamped. Provider confidence is not part
+ of the geometry; native TypeSafe evidence remains in the LM response.
+ """
+
+ model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
+
+ top: float = Field(description="Top edge as a fraction of image height, from 0 to 1.")
+ bottom: float = Field(description="Bottom edge as a fraction of image height, from 0 to 1.")
+ left: float = Field(description="Left edge as a fraction of image width, from 0 to 1.")
+ right: float = Field(description="Right edge as a fraction of image width, from 0 to 1.")
+
+ @field_validator("top", "bottom", "left", "right")
+ @classmethod
+ def validate_edge(cls, value):
+ # Runtime validation keeps the schema portable to providers that reject
+ # numeric minimum/maximum constraints in structured outputs.
+ if not 0 <= value <= 1:
+ raise ValueError("Bounding box edges must be finite numbers in [0, 1].")
+ return value
+
+ @model_validator(mode="after")
+ def validate_order(self):
+ if self.top >= self.bottom or self.left >= self.right:
+ raise ValueError("Bounding boxes require top < bottom and left < right.")
+ return self
+
+ @model_serializer(mode="wrap")
+ def serialize_model(self, handler):
+ # Geometry is structured JSON, not a media marker or a JSON-encoded string.
+ return handler(self)
+
+ def format(self) -> str:
+ return json.dumps(self.model_dump())
diff --git a/tests/adapters/test_bounding_box.py b/tests/adapters/test_bounding_box.py
new file mode 100644
index 000000000..b99f23eb1
--- /dev/null
+++ b/tests/adapters/test_bounding_box.py
@@ -0,0 +1,232 @@
+import json
+from datetime import date
+from typing import Literal
+
+import pytest
+from PIL import Image
+from pydantic import BaseModel, ValidationError
+
+import dspy
+from dspy.adapters.decision import record_evidence
+from dspy.experimental import Score
+from dspy.utils.dummies import DummyLM
+
+EDGES = {"top": 0.13, "bottom": 0.81, "left": 0.27, "right": 0.64}
+
+
+class Locate(dspy.Signature):
+ """Find the target in the photo."""
+
+ photo: dspy.Image = dspy.InputField()
+ box: dspy.BoundingBox = dspy.OutputField(desc="The dog in inputs.photo.")
+
+
+def test_geometry_round_trip_and_boundaries():
+ box = dspy.BoundingBox(**EDGES)
+ assert isinstance(box, dspy.Type)
+ assert box.model_dump() == EDGES
+ assert dspy.BoundingBox.model_validate_json(box.model_dump_json()) == box
+ assert json.loads(box.format()) == EDGES
+ assert dspy.BoundingBox(top=0, bottom=1, left=0, right=1).right == 1
+ with pytest.raises(ValidationError):
+ box.left = 0.9
+
+
[email protected]("change", [
+ {"top": -0.01}, {"right": 1.01}, {"left": float("nan")}, {"bottom": float("inf")},
+ {"top": True}, {"left": "0.27"}, {"top": 0.81}, {"left": 0.65}, {"confidence": 0.9},
+])
+def test_invalid_geometry_is_not_silently_repaired(change):
+ with pytest.raises(ValidationError):
+ dspy.BoundingBox(**{**EDGES, **change})
+
+
[email protected]("asynchronous", [False, True])
[email protected]("adapter_class", [dspy.ChatAdapter, dspy.JSONAdapter, dspy.XMLAdapter, dspy.TwoStepAdapter])
+async def test_boxes_are_structured_outputs_inputs_and_saved_demos(adapter_class, asynchronous, tmp_path):
+ class Regions(BaseModel):
+ boxes: list[dspy.BoundingBox]
+
+ signature = dspy.Signature({
+ "previous": (dspy.BoundingBox, dspy.InputField()),
+ "region": (Regions, dspy.OutputField()),
+ "missing": (dspy.BoundingBox | None, dspy.OutputField()),
+ })
+ predict = dspy.Predict(signature)
+ box = dspy.BoundingBox(**EDGES)
+ predict.demos = [dspy.Example(previous=box, region=Regions(boxes=[box]), missing=None)]
+ predict.save(tmp_path / "boxes.json")
+ predict.load(tmp_path / "boxes.json")
+ assert predict.demos[0]["previous"] == EDGES
+ assert predict.demos[0]["region"] == {"boxes": [EDGES]}
+ answer = {"region": {"boxes": [EDGES]}, "missing": None}
+ if adapter_class is dspy.TwoStepAdapter:
+ adapter = adapter_class(extraction_model=DummyLM([answer]))
+ lm = DummyLM([{"answer": "The region contains one box: top 0.13, bottom 0.81, left 0.27, right 0.64. "
+ "The missing box is null."}])
+ else:
+ adapter = adapter_class()
+ lm = DummyLM([answer], adapter=adapter)
+ with dspy.context(adapter=adapter):
+ result = await predict.acall(previous=box, lm=lm) if asynchronous else predict(previous=box, lm=lm)
+ assert result.region.boxes == [box]
+ assert result.missing is None
+ assert "CUSTOM-TYPE" not in json.dumps(lm.history[-1]["messages"])
+
+
[email protected]("left", ["0.27", "true"])
+def test_predict_boxes_from_literal_nested_xml(left, monkeypatch):
+ from tests.clients.test_lm_engine_execution import native_transport, wire
+
+ class Regions(BaseModel):
+ boxes: list[dspy.BoundingBox]
+ label: str
+
+ signature = dspy.Signature({"region": (Regions, dspy.OutputField())})
+ completion = (
+ f"<region><boxes><item><top>0.13</top><bottom>0.81</bottom><left>{left}</left>"
+ "<right>0.64</right></item></boxes><label>0.27</label></region>"
+ )
+ native_transport(monkeypatch, [wire(completion), wire(completion)])
+ lm = dspy.LM("openai/gpt-4o-mini", engine="lm15", cache=False)
+ try:
+ with dspy.context(adapter=dspy.XMLAdapter()):
+ if left == "true":
+ with pytest.raises(dspy.AdapterParseError):
+ dspy.Predict(signature)(lm=lm)
+ else:
+ result = dspy.Predict(signature)(lm=lm)
+ assert result.region.boxes == [dspy.BoundingBox(**EDGES)]
+ assert result.region.label == "0.27"
+ finally:
+ lm.close()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("mixed", [False, True])
+async def test_predict_native_boxes_evidence_images_and_cache(asynchronous, mixed, monkeypatch):
+ from dspy import lm15
+ from dspy._vendor.lm15.testing import FakeTransport
+ from tests.clients.test_lm_engine_execution import DualResponse
+
+ signature = Locate
+ answers = {"box": {"type": "box", "box": EDGES, "confidence": 0.83}}
+ if mixed:
+ signature = signature.append("second", dspy.OutputField(desc="The other target."), type_=dspy.BoundingBox)
+ signature = signature.append("visible", dspy.OutputField(desc="Is a dog visible?"), type_=bool)
+ signature = signature.append("kind", dspy.OutputField(desc="What kind of animal?"), type_=Literal["dog", "cat"])
+ signature = signature.append("size", dspy.OutputField(desc="Size, small to large."), type_=Score["small", "medium", "large"])
+ answers.update({
+ "second": {"type": "box", "box": {"top": 0.21, "bottom": 0.67, "left": 0.04, "right": 0.19}, "confidence": 0.62},
+ "visible": {"type": "noul", "noul": 0.91},
+ "kind": {"type": "choice", "choice": "dog", "probabilities": {"dog": 0.8, "cat": 0.2}, "confidence": 0.7},
+ "size": {"type": "score", "score": 1.6, "probabilities": {"0": 0.1, "1": 0.2, "2": 0.7}, "confidence": 0.6},
+ })
+ wire_response = {"model": "jev-1.14.0", "answers": answers, "usage": {"input_tokens": 31, "output_tokens": 8}}
+ transport = FakeTransport([DualResponse(200, json.dumps(wire_response).encode())])
+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", num_retries=0)
+ image = dspy.Image(Image.new("RGB", (320, 280), "red"))
+ predict = dspy.Predict(signature)
+ predict.demos = [dspy.Example(photo=image, box=dspy.BoundingBox(**EDGES))]
+ try:
+ with dspy.context(adapter=None), record_evidence() as evidence:
+ for _ in range(2):
+ result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)
+ assert result.box.model_dump() == EDGES
+ response = lm.history[-1]["response"]
+ assert response.provider_data["typesafe"]["answers"] == answers
+ assert response.data["box"] == EDGES
+ if mixed:
+ assert result.second.model_dump() == answers["second"]["box"]
+ assert result.visible is True and result.kind == "dog" and result.size.level == 2
+ assert result.size.value == pytest.approx(1.6)
+ assert set(response.probabilities) == {"visible", "kind", "size"}
+ assert response.probabilities["visible"]["true"] == 0.91
+ else:
+ assert response.probabilities is None
+ if mixed:
+ predict.fields = {"visible": {"threshold": 0.95}, "kind": {"weights": {"cat": 10}},
+ "size": {"cuts": [0.4, 1.8]}}
+ result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)
+ assert result.visible is False and result.kind == "cat" and result.size.level == 1
+ assert result.box.model_dump() == EDGES
+ predict.fields["box"] = {"threshold": 0.9}
+ with pytest.raises(ValueError, match="unsupported outputs"):
+ if asynchronous:
+ await predict.acall(photo=image, lm=lm)
+ else:
+ predict(photo=image, lm=lm)
+ assert next(answer for _, name, answer in evidence if name == "box") == answers["box"]
+ assert len(transport.requests) == 1
+ request = transport.requests[0]
+ payload = json.loads(request.body)
+ assert payload["questions"]["box"] == {"type": "box", "instructions": "The dog in inputs.photo."}
+ assert len(payload["assets"]) == 2 # input and demonstration both stay images
+ assert all(asset["data_base64"] == image.url.split(",", 1)[1] for asset in payload["assets"].values())
+ assert "Find the target" in str(payload["state"]) and "demos" in str(payload["state"])
+ assert dict(request.headers)["TypeSafe-Danger-Zone-Images"] == "true"
+ assert lm.history[-1]["usage"] == {}
+ finally:
+ await lm.aclose()
+
+
[email protected]("with_image", [False, True])
+def test_native_state_serializes_inputs_and_demos(with_image, monkeypatch):
+ from dspy import lm15
+ from dspy._vendor.lm15.testing import FakeTransport
+ from tests.clients.test_lm_engine_execution import DualResponse
+
+ signature = dspy.Signature({"metadata": (dict, dspy.InputField()), "box": (dspy.BoundingBox, dspy.OutputField())})
+ inputs = {"metadata": {"point": (9, 2), "date": date(2026, 10, 19), "tags": {"input"}}}
+ demo = {"metadata": {"point": (4, 7), "date": date(2025, 3, 8), "tags": {"demo"}}, "box": dspy.BoundingBox(**EDGES)}
+ if with_image:
+ signature = signature.prepend("photo", dspy.InputField(), type_=dspy.Image)
+ inputs["photo"] = demo["photo"] = dspy.Image(Image.new("RGB", (320, 280), "red"))
+ response = {"model": "jev-1.14.0", "answers": {"box": {"type": "box", "box": EDGES, "confidence": 0.8}}}
+ transport = FakeTransport([DualResponse(200, json.dumps(response).encode())])
+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)
+ predict = dspy.Predict(signature)
+ predict.demos = [dspy.Example(**demo)]
+ try:
+ assert predict(lm=lm, **inputs).box.model_dump() == EDGES
+ payload = json.loads(transport.requests[0].body)
+ if with_image:
+ assert len(payload["assets"]) == 2
+ else:
+ assert payload["state"]["inputs"]["metadata"] == {"point": [9, 2], "date": "2026-10-19", "tags": ["input"]}
+ assert payload["state"]["demos"][0]["metadata"] == {"point": [4, 7], "date": "2025-03-08", "tags": ["demo"]}
+ finally:
+ lm.close()
+
+
[email protected]("annotation", [list[dspy.BoundingBox], dspy.BoundingBox | None, str])
+def test_unsupported_native_outputs_are_refused(annotation, monkeypatch):
+ from dspy import lm15
+ from dspy._vendor.lm15.testing import FakeTransport
+
+ transport = FakeTransport([])
+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))
+ signature = Locate.with_updated_fields("box", type_=annotation)
+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)
+ try:
+ with dspy.context(adapter=None), pytest.raises(ValueError, match="Unsupported System One output"):
+ dspy.Predict(signature)(photo=dspy.Image(Image.new("RGB", (320, 280))), lm=lm)
+ assert not transport.requests
+ finally:
+ lm.close()
+
+
+def test_adapter_rejects_invalid_generated_geometry():
+ with pytest.raises(dspy.AdapterParseError, match="top < bottom"):
+ dspy.JSONAdapter().parse(Locate, json.dumps({"box": {**EDGES, "top": 0.9}}))
+
+ from tests.predict.test_decision_parameters import FakeClient
+
+ signature = dspy.Signature({"box": (dspy.BoundingBox, dspy.OutputField())})
+ lm = FakeClient(answers={"box": {"box": {**EDGES, "top": 0.9}, "confidence": 0.8}})
+ with pytest.raises(dspy.AdapterParseError, match="top < bottom"):
+ dspy.Predict(signature)(lm=lm)