DSPy: The framework for programming—not prompting—language models
Python
Add BoundingBox type with automatic Jev image support (#10665)
Isaac Miller6ee6ca3d427a
diff --git a/docs/docs/api/primitives/BoundingBox.md b/docs/docs/api/primitives/BoundingBox.mdnew file mode 100644index 000000000..1dcef007c--- /dev/null+++ b/docs/docs/api/primitives/BoundingBox.md@@ -0,0 +1,74 @@+# dspy.BoundingBox++`BoundingBox` is a `dspy.Type` for an axis-aligned rectangle. Its four required+fields are `top`, `bottom`, `left`, and `right`: finite numbers in [0, 1], measured+from the image's top-left corner. `top < bottom` and `left < right` are required.+Invalid geometry raises a validation error; DSPy does not clamp or reorder it.+Boxes are immutable, serialize as JSON objects, and can be input or output fields.++## Locate an object with Jev++Set `TYPESAFE_API_KEY`, then pass an image to a signature with a `BoundingBox` output:++```python+import dspy++class Locate(dspy.Signature):+ photo: dspy.Image = dspy.InputField()+ visible: bool = dspy.OutputField(desc="Is there a coffee mug in inputs.photo?")+ mug: dspy.BoundingBox = dspy.OutputField(desc="The coffee mug in inputs.photo.")++lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15")+predict = dspy.Predict(Locate)+with dspy.context(lm=lm):+ result = predict(photo=dspy.Image.from_path("kitchen.jpg"))++if result.visible:+ print(result.mug.model_dump()) # top, bottom, left, right+```++`Predict` selects the native Jev route automatically; no separate adapter is+needed. For async calls, use `await predict.acall(...)` in the same context.+There is **no not-found box**: ask a presence question and ignore the box when+the target is absent. Invalid native geometry raises `dspy.AdapterParseError`.++To convert to pixels, multiply `left` and `right` by the image width, and `top`+and `bottom` by its height. Use the dimensions of the image sent to the model;+map cropped-image coordinates back to the original yourself.++### Jev requirements and limits++- Images and boxes are experimental and require Jev 1.14 or `jev-preview`.+ DSPy sends the required experimental-image header automatically.+- Use `Predict` with one top-level box field per target. Native Jev does not+ support nested, list, or nullable boxes, generated text, tools, or streaming.+- Boxes can accompany `bool`, `Literal`, and experimental `Noul`, `Choice`, or+ `Score` outputs. Decision calibration applies to those fields, not boxes;+ use `Score` for ordinal scoring, not an integer `Literal`.+- Load PNG/JPEG bytes with `Image.from_path` or `Image.from_url`; bare URLs are+ not fetched. DSPy does not resize or apply EXIF orientation here. See the+ [image limits](https://docs.typesafe.ai/images).+- For multiple images, name the target and its input image in the output field+ description. Images in inputs and demonstrations retain their named context.++Provider confidence stays in the LM response, separate from geometry; boxes+have no entry in `response.probabilities`. Neither valid geometry nor high+confidence guarantees accurate localization. See TypeSafe's+[Box guide](https://docs.typesafe.ai/primitives/box) for reference images,+confidence thresholds, and tiling.++## Use with generative LMs++The same type works with `ChatAdapter`, `JSONAdapter`, `XMLAdapter`, and+`TwoStepAdapter`, including nested models, `list[BoundingBox]`, and optional+boxes, subject to provider support. Generated coordinates receive the same+validation but have no native TypeSafe confidence.++<!-- START_API_REF -->+::: dspy.BoundingBox+ handler: python+ options:+ show_root_heading: true+ heading_level: 2+ show_source: true+<!-- END_API_REF -->diff --git a/docs/docs/api/primitives/index.md b/docs/docs/api/primitives/index.mdindex c7102ccc8..5438d292e 100644--- a/docs/docs/api/primitives/index.md+++ b/docs/docs/api/primitives/index.md@@ -4,6 +4,7 @@ API reference for DSPy primitives. Select a page below.<!-- START_API_INDEX -->- [Audio](Audio.md)+- [BoundingBox](BoundingBox.md)- [Code](Code.md)- [Example](Example.md)- [History](History.md)diff --git a/docs/mkdocs.yml b/docs/mkdocs.ymlindex 23aa42d28..edcb74585 100644--- a/docs/mkdocs.yml+++ b/docs/mkdocs.yml@@ -156,6 +156,7 @@ nav:- Primitives:- api/primitives/index.md- Audio: api/primitives/Audio.md+ - BoundingBox: api/primitives/BoundingBox.md- Code: api/primitives/Code.md- Example: api/primitives/Example.md- History: api/primitives/History.mddiff --git a/dspy/__init__.py b/dspy/__init__.pyindex 1813f9c2b..d96c46b9d 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -10,7 +10,7 @@ from dspy.predict.flex import Flexfrom dspy.evaluate import Evaluate # isort: skipfrom dspy.clients import * # isort: skip-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, BoundingBox, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skipfrom dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skipfrom dspy.utils.exceptions import (AdapterParseError,diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.pyindex 74e199f18..70688769c 100644--- a/dspy/adapters/__init__.py+++ b/dspy/adapters/__init__.py@@ -2,7 +2,19 @@ from dspy.adapters.base import Adapterfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapterfrom dspy.adapters.two_step_adapter import TwoStepAdapter-from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCallResults, ToolCalls, Type+from dspy.adapters.types import (+ Audio,+ BoundingBox,+ Code,+ File,+ History,+ Image,+ Reasoning,+ Tool,+ ToolCallResults,+ ToolCalls,+ Type,+)from dspy.adapters.xml_adapter import XMLAdapter__all__ = [@@ -21,4 +33,5 @@ __all__ = ["ToolCalls","ToolCallResults","Reasoning",+ "BoundingBox",]diff --git a/dspy/adapters/decision.py b/dspy/adapters/decision.pyindex b1af846b2..9e25772b4 100644--- a/dspy/adapters/decision.py+++ b/dspy/adapters/decision.py@@ -7,12 +7,15 @@ import warningsfrom contextlib import contextmanagerfrom functools import lru_cache-from pydantic import BaseModel, ConfigDict, Field, create_model+from pydantic import BaseModel, ConfigDict, Field, ValidationError, create_modelfrom dspy.adapters.decision_state import DecisionState+from dspy.adapters.types.base_type import split_message_content_for_custom_types+from dspy.adapters.types.bounding_box import BoundingBoxfrom dspy.adapters.types.decision import Choice, Noul, Probability, Scorefrom dspy.dsp.utils.settings import settings-from dspy.lm15 import Config, DataPart, Message, Request, choice, score, yes_no+from dspy.lm15 import Config, DataPart, Message, Request, bounding_box, choice, request_from_openai_chat, score, yes_no+from dspy.utils.exceptions import AdapterParseError@contextmanager@@ -63,7 +66,7 @@ def resolve_adapter(lm, adapter, signature, fields, declared_signature=None):UserWarning,stacklevel=2,)- return DecisionAdapter(adapter, state, system_one) if state.types else adapter+ return DecisionAdapter(adapter, state, system_one) if system_one or state.types else adapterclass DecisionAdapter:@@ -75,7 +78,7 @@ class DecisionAdapter:self.system_one = system_onedef _prepare(self, lm, signature, demos, inputs, lm_kwargs):- from dspy.adapters.utils import get_field_description_string+ from dspy.adapters.utils import get_field_description_string, serialize_for_jsonfrom dspy.predict.predict import serialize_objectif settings.send_stream is not None:@@ -90,12 +93,11 @@ class DecisionAdapter:state = {"instructions": signature.instructions,"input_fields": get_field_description_string(signature.input_fields),- "inputs": serialize_object({k: v for k, v in inputs.items() if k in signature.input_fields}),+ "inputs": {k: v for k, v in inputs.items() if k in signature.input_fields},}if demos:- state["demos"] = [- serialize_object({k: v for k, v in demo.items() if k in signature.fields}) for demo in demos- ]+ state["demos"] = [{k: v for k, v in demo.items() if k in signature.fields} for demo in demos]+ state = serialize_for_json(state)properties = {}for name, kind in types.items():if issubclass(kind, Noul):@@ -110,10 +112,24 @@ class DecisionAdapter:"use a nonempty label such as 'no_match'.")properties[name] = choice(name, list(criteria))+ for name, field in signature.output_fields.items():+ if field.annotation is BoundingBox:+ desc = field.json_schema_extra.get("desc", "")+ instructions = name if desc == f"${{{name}}}" else desc or name+ properties[name] = bounding_box(instructions)+ questions[name] = {"type": "box", "instructions": instructions}+ # Keep text-only state structured. For media, reuse DSPy's marker+ # expansion and lm15's conversion rather than encoding images here.+ messages = split_message_content_for_custom_types([+ {"role": "user", "content": json.dumps(state, ensure_ascii=False)}+ ])+ message = Message.user(DataPart(value=state))+ if isinstance(messages[0]["content"], list):+ message = request_from_openai_chat({"model": lm.model, "messages": messages}).messages[0]# The schema declares answer spaces; native questions retain structured instructions and criteria.response_format = {"type": "json_schema", "schema": {"type": "object", "properties": properties}}return Request(- model=lm.model, messages=(Message.user(DataPart(value=state)),),+ model=lm.model, messages=(message,),config=Config(response_format=response_format, extensions={"questions": questions}),)for name, kind in types.items():@@ -134,6 +150,18 @@ class DecisionAdapter:results = []for completion in completions:answers = {}+ boxes = {}+ if self.system_one:+ for name, field in self.state.signature.output_fields.items():+ if field.annotation is BoundingBox:+ answers[name] = copy.deepcopy(completion[name])+ try:+ boxes[name] = BoundingBox.model_validate(answers[name]["box"])+ except ValidationError as error:+ raise AdapterParseError(+ adapter_name="DecisionAdapter", signature=self.state.signature,+ lm_response=json.dumps(completion), message=str(error),+ ) from errorfor name, kind in types.items():answer = completion[name]if isinstance(answer, BaseModel):@@ -152,7 +180,7 @@ class DecisionAdapter:caller = (settings.caller_modules or [None])[-1]with lock:log.extend((caller, name, copy.deepcopy(answer)) for name, answer in answers.items())- results.append({**completion, **self.state._decode(answers)})+ results.append({**completion, **boxes, **self.state._decode(answers)})return resultsdef __call__(self, lm, lm_kwargs, signature, demos, inputs):diff --git a/dspy/adapters/decision_state.py b/dspy/adapters/decision_state.pyindex 198567840..dc7f746f0 100644--- a/dspy/adapters/decision_state.py+++ b/dspy/adapters/decision_state.py@@ -8,6 +8,7 @@ from typing import get_argsfrom pydantic import JsonValue, TypeAdapter+from dspy.adapters.types.bounding_box import BoundingBoxfrom dspy.adapters.types.decision import Choice, Noul, Score, decision_type_JSON_ADAPTER = TypeAdapter(JsonValue)@@ -27,6 +28,9 @@ class DecisionState:self.types = {}self.fields = {}for name, field in signature.output_fields.items():+ # Native boxes carry geometry, not calibratable decision values.+ if system_one and field.annotation is BoundingBox:+ continueif not system_one and name not in fields and not any(isinstance(annotation, type) and issubclass(annotation, (Noul, Score, Choice))for annotation in (field.annotation, *field.metadata)diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.pyindex 36403e74d..9ea5024a4 100644--- a/dspy/adapters/types/__init__.py+++ b/dspy/adapters/types/__init__.py@@ -1,5 +1,6 @@from dspy.adapters.types.audio import Audiofrom dspy.adapters.types.base_type import Type+from dspy.adapters.types.bounding_box import BoundingBoxfrom dspy.adapters.types.code import Codefrom dspy.adapters.types.file import Filefrom dspy.adapters.types.history import History@@ -7,4 +8,4 @@ from dspy.adapters.types.image import Imagefrom dspy.adapters.types.reasoning import Reasoningfrom dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls-__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "ToolCallResults", "Code", "Reasoning"]+__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "ToolCallResults", "Code", "Reasoning", "BoundingBox"]diff --git a/dspy/adapters/types/bounding_box.py b/dspy/adapters/types/bounding_box.pynew file mode 100644index 000000000..73f0520ba--- /dev/null+++ b/dspy/adapters/types/bounding_box.py@@ -0,0 +1,49 @@+import json++from pydantic import ConfigDict, Field, field_validator, model_serializer, model_validator++from dspy.adapters.types.base_type import Type+++class BoundingBox(Type):+ """An axis-aligned rectangle in normalized image coordinates.++ Edges are finite numbers in [0, 1], measured from the top-left corner.+ Multiply left/right by the image width and top/bottom by its height to+ obtain pixel coordinates. Use the dimensions of the upright image actually+ sent to the model, not those of an original image before resizing/cropping.++ Boxes have positive area: top < bottom and left < right. Invalid outputs+ are rejected, never reordered or clamped. Provider confidence is not part+ of the geometry; native TypeSafe evidence remains in the LM response.+ """++ model_config = ConfigDict(frozen=True, extra="forbid", strict=True)++ top: float = Field(description="Top edge as a fraction of image height, from 0 to 1.")+ bottom: float = Field(description="Bottom edge as a fraction of image height, from 0 to 1.")+ left: float = Field(description="Left edge as a fraction of image width, from 0 to 1.")+ right: float = Field(description="Right edge as a fraction of image width, from 0 to 1.")++ @field_validator("top", "bottom", "left", "right")+ @classmethod+ def validate_edge(cls, value):+ # Runtime validation keeps the schema portable to providers that reject+ # numeric minimum/maximum constraints in structured outputs.+ if not 0 <= value <= 1:+ raise ValueError("Bounding box edges must be finite numbers in [0, 1].")+ return value++ @model_validator(mode="after")+ def validate_order(self):+ if self.top >= self.bottom or self.left >= self.right:+ raise ValueError("Bounding boxes require top < bottom and left < right.")+ return self++ @model_serializer(mode="wrap")+ def serialize_model(self, handler):+ # Geometry is structured JSON, not a media marker or a JSON-encoded string.+ return handler(self)++ def format(self) -> str:+ return json.dumps(self.model_dump())diff --git a/tests/adapters/test_bounding_box.py b/tests/adapters/test_bounding_box.pynew file mode 100644index 000000000..b99f23eb1--- /dev/null+++ b/tests/adapters/test_bounding_box.py@@ -0,0 +1,232 @@+import json+from datetime import date+from typing import Literal++import pytest+from PIL import Image+from pydantic import BaseModel, ValidationError++import dspy+from dspy.adapters.decision import record_evidence+from dspy.experimental import Score+from dspy.utils.dummies import DummyLM++EDGES = {"top": 0.13, "bottom": 0.81, "left": 0.27, "right": 0.64}+++class Locate(dspy.Signature):+ """Find the target in the photo."""++ photo: dspy.Image = dspy.InputField()+ box: dspy.BoundingBox = dspy.OutputField(desc="The dog in inputs.photo.")+++def test_geometry_round_trip_and_boundaries():+ box = dspy.BoundingBox(**EDGES)+ assert isinstance(box, dspy.Type)+ assert box.model_dump() == EDGES+ assert dspy.BoundingBox.model_validate_json(box.model_dump_json()) == box+ assert json.loads(box.format()) == EDGES+ assert dspy.BoundingBox(top=0, bottom=1, left=0, right=1).right == 1+ with pytest.raises(ValidationError):+ box.left = 0.9++[email protected]("change", [+ {"top": -0.01}, {"right": 1.01}, {"left": float("nan")}, {"bottom": float("inf")},+ {"top": True}, {"left": "0.27"}, {"top": 0.81}, {"left": 0.65}, {"confidence": 0.9},+])+def test_invalid_geometry_is_not_silently_repaired(change):+ with pytest.raises(ValidationError):+ dspy.BoundingBox(**{**EDGES, **change})++[email protected]("asynchronous", [False, True])[email protected]("adapter_class", [dspy.ChatAdapter, dspy.JSONAdapter, dspy.XMLAdapter, dspy.TwoStepAdapter])+async def test_boxes_are_structured_outputs_inputs_and_saved_demos(adapter_class, asynchronous, tmp_path):+ class Regions(BaseModel):+ boxes: list[dspy.BoundingBox]++ signature = dspy.Signature({+ "previous": (dspy.BoundingBox, dspy.InputField()),+ "region": (Regions, dspy.OutputField()),+ "missing": (dspy.BoundingBox | None, dspy.OutputField()),+ })+ predict = dspy.Predict(signature)+ box = dspy.BoundingBox(**EDGES)+ predict.demos = [dspy.Example(previous=box, region=Regions(boxes=[box]), missing=None)]+ predict.save(tmp_path / "boxes.json")+ predict.load(tmp_path / "boxes.json")+ assert predict.demos[0]["previous"] == EDGES+ assert predict.demos[0]["region"] == {"boxes": [EDGES]}+ answer = {"region": {"boxes": [EDGES]}, "missing": None}+ if adapter_class is dspy.TwoStepAdapter:+ adapter = adapter_class(extraction_model=DummyLM([answer]))+ lm = DummyLM([{"answer": "The region contains one box: top 0.13, bottom 0.81, left 0.27, right 0.64. "+ "The missing box is null."}])+ else:+ adapter = adapter_class()+ lm = DummyLM([answer], adapter=adapter)+ with dspy.context(adapter=adapter):+ result = await predict.acall(previous=box, lm=lm) if asynchronous else predict(previous=box, lm=lm)+ assert result.region.boxes == [box]+ assert result.missing is None+ assert "CUSTOM-TYPE" not in json.dumps(lm.history[-1]["messages"])++[email protected]("left", ["0.27", "true"])+def test_predict_boxes_from_literal_nested_xml(left, monkeypatch):+ from tests.clients.test_lm_engine_execution import native_transport, wire++ class Regions(BaseModel):+ boxes: list[dspy.BoundingBox]+ label: str++ signature = dspy.Signature({"region": (Regions, dspy.OutputField())})+ completion = (+ f"<region><boxes><item><top>0.13</top><bottom>0.81</bottom><left>{left}</left>"+ "<right>0.64</right></item></boxes><label>0.27</label></region>"+ )+ native_transport(monkeypatch, [wire(completion), wire(completion)])+ lm = dspy.LM("openai/gpt-4o-mini", engine="lm15", cache=False)+ try:+ with dspy.context(adapter=dspy.XMLAdapter()):+ if left == "true":+ with pytest.raises(dspy.AdapterParseError):+ dspy.Predict(signature)(lm=lm)+ else:+ result = dspy.Predict(signature)(lm=lm)+ assert result.region.boxes == [dspy.BoundingBox(**EDGES)]+ assert result.region.label == "0.27"+ finally:+ lm.close()++[email protected]("asynchronous", [False, True])[email protected]("mixed", [False, True])+async def test_predict_native_boxes_evidence_images_and_cache(asynchronous, mixed, monkeypatch):+ from dspy import lm15+ from dspy._vendor.lm15.testing import FakeTransport+ from tests.clients.test_lm_engine_execution import DualResponse++ signature = Locate+ answers = {"box": {"type": "box", "box": EDGES, "confidence": 0.83}}+ if mixed:+ signature = signature.append("second", dspy.OutputField(desc="The other target."), type_=dspy.BoundingBox)+ signature = signature.append("visible", dspy.OutputField(desc="Is a dog visible?"), type_=bool)+ signature = signature.append("kind", dspy.OutputField(desc="What kind of animal?"), type_=Literal["dog", "cat"])+ signature = signature.append("size", dspy.OutputField(desc="Size, small to large."), type_=Score["small", "medium", "large"])+ answers.update({+ "second": {"type": "box", "box": {"top": 0.21, "bottom": 0.67, "left": 0.04, "right": 0.19}, "confidence": 0.62},+ "visible": {"type": "noul", "noul": 0.91},+ "kind": {"type": "choice", "choice": "dog", "probabilities": {"dog": 0.8, "cat": 0.2}, "confidence": 0.7},+ "size": {"type": "score", "score": 1.6, "probabilities": {"0": 0.1, "1": 0.2, "2": 0.7}, "confidence": 0.6},+ })+ wire_response = {"model": "jev-1.14.0", "answers": answers, "usage": {"input_tokens": 31, "output_tokens": 8}}+ transport = FakeTransport([DualResponse(200, json.dumps(wire_response).encode())])+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", num_retries=0)+ image = dspy.Image(Image.new("RGB", (320, 280), "red"))+ predict = dspy.Predict(signature)+ predict.demos = [dspy.Example(photo=image, box=dspy.BoundingBox(**EDGES))]+ try:+ with dspy.context(adapter=None), record_evidence() as evidence:+ for _ in range(2):+ result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)+ assert result.box.model_dump() == EDGES+ response = lm.history[-1]["response"]+ assert response.provider_data["typesafe"]["answers"] == answers+ assert response.data["box"] == EDGES+ if mixed:+ assert result.second.model_dump() == answers["second"]["box"]+ assert result.visible is True and result.kind == "dog" and result.size.level == 2+ assert result.size.value == pytest.approx(1.6)+ assert set(response.probabilities) == {"visible", "kind", "size"}+ assert response.probabilities["visible"]["true"] == 0.91+ else:+ assert response.probabilities is None+ if mixed:+ predict.fields = {"visible": {"threshold": 0.95}, "kind": {"weights": {"cat": 10}},+ "size": {"cuts": [0.4, 1.8]}}+ result = await predict.acall(photo=image, lm=lm) if asynchronous else predict(photo=image, lm=lm)+ assert result.visible is False and result.kind == "cat" and result.size.level == 1+ assert result.box.model_dump() == EDGES+ predict.fields["box"] = {"threshold": 0.9}+ with pytest.raises(ValueError, match="unsupported outputs"):+ if asynchronous:+ await predict.acall(photo=image, lm=lm)+ else:+ predict(photo=image, lm=lm)+ assert next(answer for _, name, answer in evidence if name == "box") == answers["box"]+ assert len(transport.requests) == 1+ request = transport.requests[0]+ payload = json.loads(request.body)+ assert payload["questions"]["box"] == {"type": "box", "instructions": "The dog in inputs.photo."}+ assert len(payload["assets"]) == 2 # input and demonstration both stay images+ assert all(asset["data_base64"] == image.url.split(",", 1)[1] for asset in payload["assets"].values())+ assert "Find the target" in str(payload["state"]) and "demos" in str(payload["state"])+ assert dict(request.headers)["TypeSafe-Danger-Zone-Images"] == "true"+ assert lm.history[-1]["usage"] == {}+ finally:+ await lm.aclose()++[email protected]("with_image", [False, True])+def test_native_state_serializes_inputs_and_demos(with_image, monkeypatch):+ from dspy import lm15+ from dspy._vendor.lm15.testing import FakeTransport+ from tests.clients.test_lm_engine_execution import DualResponse++ signature = dspy.Signature({"metadata": (dict, dspy.InputField()), "box": (dspy.BoundingBox, dspy.OutputField())})+ inputs = {"metadata": {"point": (9, 2), "date": date(2026, 10, 19), "tags": {"input"}}}+ demo = {"metadata": {"point": (4, 7), "date": date(2025, 3, 8), "tags": {"demo"}}, "box": dspy.BoundingBox(**EDGES)}+ if with_image:+ signature = signature.prepend("photo", dspy.InputField(), type_=dspy.Image)+ inputs["photo"] = demo["photo"] = dspy.Image(Image.new("RGB", (320, 280), "red"))+ response = {"model": "jev-1.14.0", "answers": {"box": {"type": "box", "box": EDGES, "confidence": 0.8}}}+ transport = FakeTransport([DualResponse(200, json.dumps(response).encode())])+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)+ predict = dspy.Predict(signature)+ predict.demos = [dspy.Example(**demo)]+ try:+ assert predict(lm=lm, **inputs).box.model_dump() == EDGES+ payload = json.loads(transport.requests[0].body)+ if with_image:+ assert len(payload["assets"]) == 2+ else:+ assert payload["state"]["inputs"]["metadata"] == {"point": [9, 2], "date": "2026-10-19", "tags": ["input"]}+ assert payload["state"]["demos"][0]["metadata"] == {"point": [4, 7], "date": "2025-03-08", "tags": ["demo"]}+ finally:+ lm.close()++[email protected]("annotation", [list[dspy.BoundingBox], dspy.BoundingBox | None, str])+def test_unsupported_native_outputs_are_refused(annotation, monkeypatch):+ from dspy import lm15+ from dspy._vendor.lm15.testing import FakeTransport++ transport = FakeTransport([])+ monkeypatch.setattr("dspy.clients.execution.RouterConfig", lambda **kwargs: lm15.RouterConfig(**{**kwargs, "transport": transport}))+ signature = Locate.with_updated_fields("box", type_=annotation)+ lm = dspy.LM("typesafe:jev-1.14.0", engine="lm15", api_key="test", cache=False, num_retries=0)+ try:+ with dspy.context(adapter=None), pytest.raises(ValueError, match="Unsupported System One output"):+ dspy.Predict(signature)(photo=dspy.Image(Image.new("RGB", (320, 280))), lm=lm)+ assert not transport.requests+ finally:+ lm.close()+++def test_adapter_rejects_invalid_generated_geometry():+ with pytest.raises(dspy.AdapterParseError, match="top < bottom"):+ dspy.JSONAdapter().parse(Locate, json.dumps({"box": {**EDGES, "top": 0.9}}))++ from tests.predict.test_decision_parameters import FakeClient++ signature = dspy.Signature({"box": (dspy.BoundingBox, dspy.OutputField())})+ lm = FakeClient(answers={"box": {"box": {**EDGES, "top": 0.9}, "confidence": 0.8}})+ with pytest.raises(dspy.AdapterParseError, match="top < bottom"):+ dspy.Predict(signature)(lm=lm)