Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix `Literal` tests

Lukas Kreussele575746eceda
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index baf19b078..2b39cd137 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -116,10 +116,12 @@ class TypedPredictor(dspy.Module):
"""Return a string representation of the TypedPredictor object."""
return f"TypedPredictor({self.signature})"
- def _make_example(self, type_) -> str:
+ def _make_example(self, field) -> str:
# Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.
-
- schema = json.dumps(type_.model_json_schema())
+ if hasattr(field, "model_json_schema"):
+ pass
+ schema = field.json_schema_extra["schema"]
+ parser = field.json_schema_extra["parser"]
if self.wrap_json:
schema = "```json\n" + schema + "\n```\n"
json_object = dspy.Predict(
@@ -128,9 +130,9 @@ class TypedPredictor(dspy.Module):
"Make a very succinct json object that validates with the following schema",
),
)(json_schema=schema).json_object
- # We use the model_validate_json method to make sure the example is valid
+ # We use the parser to make sure the json object is valid.
try:
- type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))
+ parser(_unwrap_json(json_object, parser))
except (pydantic.ValidationError, ValueError):
return "" # Unable to make an example
return json_object
@@ -228,19 +230,19 @@ class TypedPredictor(dspy.Module):
)
else:
# Anything else we wrap in a pydantic object
- if (inspect.isclass(type_)
+ if (
+ inspect.isclass(type_)
and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9
- and issubclass(type_, pydantic.BaseModel)):
-
+ and issubclass(type_, pydantic.BaseModel)
+ ):
to_json = lambda x: x.model_dump_json()
from_json = lambda x, type_=type_: type_.model_validate_json(x)
schema = json.dumps(type_.model_json_schema())
else:
adapter = pydantic.TypeAdapter(type_)
- to_json = lambda x : adapter.serializer.to_json(x)
+ to_json = lambda x: adapter.serializer.to_json(x)
from_json = lambda x, type_=adapter: type_.validate_json(x)
schema = json.dumps(adapter.json_schema())
-
if self.wrap_json:
to_json = lambda x, inner=to_json: "```json\n" + inner(x) + "\n```\n"
schema = "```json\n" + schema + "\n```"
@@ -250,6 +252,7 @@ class TypedPredictor(dspy.Module):
+ (". Respond with a single JSON object. JSON Schema: " + schema),
format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),
parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),
+ schema=schema,
type_=type_,
)
else: # If input field
@@ -311,7 +314,7 @@ class TypedPredictor(dspy.Module):
if (
try_i + 1 < self.max_retries
and prefix not in current_desc
- and (example := self._make_example(field.annotation))
+ and (example := self._make_example(field))
):
signature = signature.with_updated_fields(
name,
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 7b5c032ad..9e84d8393 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -557,7 +557,7 @@ def test_parse_type_string():
def test_literal():
- lm = DummyLM([f'"{i}"' for i in range(100)])
+ lm = DummyLM(['"2"', '"3"'])
dspy.settings.configure(lm=lm)
@predictor
@@ -567,8 +567,22 @@ def test_literal():
assert f() == "2"
+def test_literal_missmatch():
+ lm = DummyLM([f'"{i}"' for i in range(5, 100)])
+ dspy.settings.configure(lm=lm)
+
+ @predictor(max_retries=1)
+ def f() -> Literal["2", "3"]:
+ pass
+
+ with pytest.raises(Exception) as e_info:
+ f()
+
+ assert e_info.value.args[1]["f"] == "Input should be '2' or '3': (error type: literal_error)"
+
+
def test_literal_int():
- lm = DummyLM([f'{i}' for i in range(100)])
+ lm = DummyLM(["2", "3"])
dspy.settings.configure(lm=lm)
@predictor
@@ -578,6 +592,20 @@ def test_literal_int():
assert f() == 2
+def test_literal_int_missmatch():
+ lm = DummyLM([f"{i}" for i in range(5, 100)])
+ dspy.settings.configure(lm=lm)
+
+ @predictor(max_retries=1)
+ def f() -> Literal[2, 3]:
+ pass
+
+ with pytest.raises(Exception) as e_info:
+ f()
+
+ assert e_info.value.args[1]["f"] == "Input should be 2 or 3: (error type: literal_error)"
+
+
def test_fields_on_base_signature():
class SimpleOutput(dspy.Signature):
output: float = dspy.OutputField(gt=0, lt=1)
@@ -896,9 +924,7 @@ def _test_demos_missing_input():
def test_conlist():
- dspy.settings.configure(
- lm=DummyLM(['[]', '[1]', '[1, 2]', '[1, 2, 3]'])
- )
+ dspy.settings.configure(lm=DummyLM(["[]", "[1]", "[1, 2]", "[1, 2, 3]"]))
@predictor
def make_numbers(input: str) -> Annotated[list[int], Field(min_items=2)]:
@@ -908,9 +934,7 @@ def test_conlist():
def test_conlist2():
- dspy.settings.configure(
- lm=DummyLM(['[]', '[1]', '[1, 2]', '[1, 2, 3]'])
- )
+ dspy.settings.configure(lm=DummyLM(["[]", "[1]", "[1, 2]", "[1, 2, 3]"]))
make_numbers = TypedPredictor("input:str -> output:Annotated[List[int], Field(min_items=2)]")
assert make_numbers(input="What are the first two numbers?").output == [1, 2]
diff --git a/tests/functional/test_signature_typed.py b/tests/functional/test_signature_typed.py
index c3d4ffb8d..39eb8c2ff 100644
--- a/tests/functional/test_signature_typed.py
+++ b/tests/functional/test_signature_typed.py
@@ -6,17 +6,20 @@ import dspy
from dspy.functional import TypedPredictor
from dspy.signatures.signature import signature_to_template
-def get_field_and_parser(signature:dspy.Signature) -> tuple[Any, Any]:
+
+def get_field_and_parser(signature: dspy.Signature) -> tuple[Any, Any]:
module = TypedPredictor(signature)
signature = module._prepare_signature()
assert "answer" in signature.fields, "'answer' not in signature.fields"
- field = signature.fields.get('answer')
+ field = signature.fields.get("answer")
parser = field.json_schema_extra.get("parser")
return field, parser
-
+
+
class Mysubmodel(pydantic.BaseModel):
sub_floating: float
-
+
+
class MyModel(pydantic.BaseModel):
floating: float
string: str
@@ -28,7 +31,8 @@ class MyModel(pydantic.BaseModel):
submodel: Mysubmodel
optional_submodel: Optional[Mysubmodel]
optional_existing_submodule: Optional[Mysubmodel]
-
+
+
def build_model_instance() -> MyModel:
return MyModel(
floating=3.14,
@@ -42,101 +46,109 @@ def build_model_instance() -> MyModel:
optional_submodel=None,
optional_existing_submodule=Mysubmodel(sub_floating=42.42),
)
-
[email protected]("test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)])
-def test_basic_types(test_type:type, serialized:str, expected:Any):
+
+ "test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)]
+)
+def test_basic_types(test_type: type, serialized: str, expected: Any):
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: test_type = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
assert parser is test_type, "Parser is not correct for 'answer'"
assert parser(serialized) == expected, f"{test_type}({serialized})!= {expected}"
-
-
-
+
+
def test_boolean():
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: bool = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
assert parser("true"), f"Parsing 'true' failed"
assert not parser("false"), f"Parsing 'false' failed"
-
-
[email protected]("test_type,serialized, expected", [(list[str], '["foo", "bar"]', ['foo', 'bar']), (tuple[int, float], '[42, 3.14]', (42, 3.14))])
-def test_sequences(test_type:type, serialized:str, expected:Any):
+
+
+ "test_type,serialized, expected",
+ [(list[str], '["foo", "bar"]', ["foo", "bar"]), (tuple[int, float], "[42, 3.14]", (42, 3.14))],
+)
+def test_sequences(test_type: type, serialized: str, expected: Any):
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: test_type = dspy.OutputField()
-
-
+
_, parser = get_field_and_parser(MySignature)
-
+
assert parser(serialized) == expected, f"Parsing {expected} failed"
-
[email protected]("test_type,serialized, expected", [
- (Optional[str], '"foobar"', "foobar"),
- (Optional[str], "null", None),
- (Union[str, float], "3.14", 3.14),
- (Union[str, bool], "true", True)
- ])
-def test_unions(test_type:type, serialized:str, expected:Any):
+
+
+ "test_type,serialized, expected",
+ [
+ (Optional[str], '"foobar"', "foobar"),
+ (Optional[str], "null", None),
+ (Union[str, float], "3.14", 3.14),
+ (Union[str, bool], "true", True),
+ ],
+)
+def test_unions(test_type: type, serialized: str, expected: Any):
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: test_type = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
-
+
assert parser(serialized) == expected, f"Parsing {expected} failed"
-
-
+
+
def test_pydantic():
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: MyModel = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
-
+
instance = build_model_instance()
parsed_instance = parser(instance.model_dump_json())
-
+
assert parsed_instance == instance, f"{instance} != {parsed_instance}"
-
-def test_optional_pydantic():
+
+
+def test_optional_pydantic():
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: Optional[MyModel] = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
-
+
instance = build_model_instance()
parsed_instance = parser(instance.model_dump_json())
assert parsed_instance == instance, f"{instance} != {parsed_instance}"
-
- #Check null case
+
+ # Check null case
parsed_instance = parser("null")
- assert parsed_instance == None, "Optional[MyModel] should be None"
-
+ assert parsed_instance == None, "Optional[MyModel] should be None"
+
+
def test_dataclass():
-
from dataclasses import dataclass
-
- @dataclass(frozen = True)
+
+ @dataclass(frozen=True)
class MyDataclass:
string: str
number: int
floating: float
boolean: bool
-
+
class MySignature(dspy.Signature):
question: str = dspy.InputField()
answer: MyDataclass = dspy.OutputField()
-
+
_, parser = get_field_and_parser(MySignature)
-
+
instance = MyDataclass("foobar", 42, 3.14, True)
parsed_instance = parser('{"string": "foobar", "number": 42, "floating": 3.14, "boolean": true}')
- assert parsed_instance == instance, f"{instance} != {parsed_instance}"
\ No newline at end of file
+ assert parsed_instance == instance, f"{instance} != {parsed_instance}"