DSPy: The framework for programming—not prompting—language models
Python
Fix `Literal` tests
Lukas Kreussele575746eceda
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.pyindex baf19b078..2b39cd137 100644--- a/dspy/functional/functional.py+++ b/dspy/functional/functional.py@@ -116,10 +116,12 @@ class TypedPredictor(dspy.Module):"""Return a string representation of the TypedPredictor object."""return f"TypedPredictor({self.signature})"- def _make_example(self, type_) -> str:+ def _make_example(self, field) -> str:# Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.-- schema = json.dumps(type_.model_json_schema())+ if hasattr(field, "model_json_schema"):+ pass+ schema = field.json_schema_extra["schema"]+ parser = field.json_schema_extra["parser"]if self.wrap_json:schema = "```json\n" + schema + "\n```\n"json_object = dspy.Predict(@@ -128,9 +130,9 @@ class TypedPredictor(dspy.Module):"Make a very succinct json object that validates with the following schema",),)(json_schema=schema).json_object- # We use the model_validate_json method to make sure the example is valid+ # We use the parser to make sure the json object is valid.try:- type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))+ parser(_unwrap_json(json_object, parser))except (pydantic.ValidationError, ValueError):return "" # Unable to make an examplereturn json_object@@ -228,19 +230,19 @@ class TypedPredictor(dspy.Module):)else:# Anything else we wrap in a pydantic object- if (inspect.isclass(type_)+ if (+ inspect.isclass(type_)and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9- and issubclass(type_, pydantic.BaseModel)):-+ and issubclass(type_, pydantic.BaseModel)+ ):to_json = lambda x: x.model_dump_json()from_json = lambda x, type_=type_: type_.model_validate_json(x)schema = json.dumps(type_.model_json_schema())else:adapter = pydantic.TypeAdapter(type_)- to_json = lambda x : adapter.serializer.to_json(x)+ to_json = lambda x: adapter.serializer.to_json(x)from_json = lambda x, type_=adapter: type_.validate_json(x)schema = json.dumps(adapter.json_schema())-if self.wrap_json:to_json = lambda x, inner=to_json: "```json\n" + inner(x) + "\n```\n"schema = "```json\n" + schema + "\n```"@@ -250,6 +252,7 @@ class TypedPredictor(dspy.Module):+ (". Respond with a single JSON object. JSON Schema: " + schema),format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),+ schema=schema,type_=type_,)else: # If input field@@ -311,7 +314,7 @@ class TypedPredictor(dspy.Module):if (try_i + 1 < self.max_retriesand prefix not in current_desc- and (example := self._make_example(field.annotation))+ and (example := self._make_example(field))):signature = signature.with_updated_fields(name,diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.pyindex 7b5c032ad..9e84d8393 100644--- a/tests/functional/test_functional.py+++ b/tests/functional/test_functional.py@@ -557,7 +557,7 @@ def test_parse_type_string():def test_literal():- lm = DummyLM([f'"{i}"' for i in range(100)])+ lm = DummyLM(['"2"', '"3"'])dspy.settings.configure(lm=lm)@predictor@@ -567,8 +567,22 @@ def test_literal():assert f() == "2"+def test_literal_missmatch():+ lm = DummyLM([f'"{i}"' for i in range(5, 100)])+ dspy.settings.configure(lm=lm)++ @predictor(max_retries=1)+ def f() -> Literal["2", "3"]:+ pass++ with pytest.raises(Exception) as e_info:+ f()++ assert e_info.value.args[1]["f"] == "Input should be '2' or '3': (error type: literal_error)"++def test_literal_int():- lm = DummyLM([f'{i}' for i in range(100)])+ lm = DummyLM(["2", "3"])dspy.settings.configure(lm=lm)@predictor@@ -578,6 +592,20 @@ def test_literal_int():assert f() == 2+def test_literal_int_missmatch():+ lm = DummyLM([f"{i}" for i in range(5, 100)])+ dspy.settings.configure(lm=lm)++ @predictor(max_retries=1)+ def f() -> Literal[2, 3]:+ pass++ with pytest.raises(Exception) as e_info:+ f()++ assert e_info.value.args[1]["f"] == "Input should be 2 or 3: (error type: literal_error)"++def test_fields_on_base_signature():class SimpleOutput(dspy.Signature):output: float = dspy.OutputField(gt=0, lt=1)@@ -896,9 +924,7 @@ def _test_demos_missing_input():def test_conlist():- dspy.settings.configure(- lm=DummyLM(['[]', '[1]', '[1, 2]', '[1, 2, 3]'])- )+ dspy.settings.configure(lm=DummyLM(["[]", "[1]", "[1, 2]", "[1, 2, 3]"]))@predictordef make_numbers(input: str) -> Annotated[list[int], Field(min_items=2)]:@@ -908,9 +934,7 @@ def test_conlist():def test_conlist2():- dspy.settings.configure(- lm=DummyLM(['[]', '[1]', '[1, 2]', '[1, 2, 3]'])- )+ dspy.settings.configure(lm=DummyLM(["[]", "[1]", "[1, 2]", "[1, 2, 3]"]))make_numbers = TypedPredictor("input:str -> output:Annotated[List[int], Field(min_items=2)]")assert make_numbers(input="What are the first two numbers?").output == [1, 2]diff --git a/tests/functional/test_signature_typed.py b/tests/functional/test_signature_typed.pyindex c3d4ffb8d..39eb8c2ff 100644--- a/tests/functional/test_signature_typed.py+++ b/tests/functional/test_signature_typed.py@@ -6,17 +6,20 @@ import dspyfrom dspy.functional import TypedPredictorfrom dspy.signatures.signature import signature_to_template-def get_field_and_parser(signature:dspy.Signature) -> tuple[Any, Any]:++def get_field_and_parser(signature: dspy.Signature) -> tuple[Any, Any]:module = TypedPredictor(signature)signature = module._prepare_signature()assert "answer" in signature.fields, "'answer' not in signature.fields"- field = signature.fields.get('answer')+ field = signature.fields.get("answer")parser = field.json_schema_extra.get("parser")return field, parser-++class Mysubmodel(pydantic.BaseModel):sub_floating: float-++class MyModel(pydantic.BaseModel):floating: floatstring: str@@ -28,7 +31,8 @@ class MyModel(pydantic.BaseModel):submodel: Mysubmodeloptional_submodel: Optional[Mysubmodel]optional_existing_submodule: Optional[Mysubmodel]-++def build_model_instance() -> MyModel:return MyModel(floating=3.14,@@ -42,101 +46,109 @@ def build_model_instance() -> MyModel:optional_submodel=None,optional_existing_submodule=Mysubmodel(sub_floating=42.42),)-[email protected]("test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)])-def test_basic_types(test_type:type, serialized:str, expected:Any):++ "test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)]+)+def test_basic_types(test_type: type, serialized: str, expected: Any):class MySignature(dspy.Signature):question: str = dspy.InputField()answer: test_type = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)assert parser is test_type, "Parser is not correct for 'answer'"assert parser(serialized) == expected, f"{test_type}({serialized})!= {expected}"---++def test_boolean():class MySignature(dspy.Signature):question: str = dspy.InputField()answer: bool = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)assert parser("true"), f"Parsing 'true' failed"assert not parser("false"), f"Parsing 'false' failed"--[email protected]("test_type,serialized, expected", [(list[str], '["foo", "bar"]', ['foo', 'bar']), (tuple[int, float], '[42, 3.14]', (42, 3.14))])-def test_sequences(test_type:type, serialized:str, expected:Any):+++ "test_type,serialized, expected",+ [(list[str], '["foo", "bar"]', ["foo", "bar"]), (tuple[int, float], "[42, 3.14]", (42, 3.14))],+)+def test_sequences(test_type: type, serialized: str, expected: Any):class MySignature(dspy.Signature):question: str = dspy.InputField()answer: test_type = dspy.OutputField()--+_, parser = get_field_and_parser(MySignature)-+assert parser(serialized) == expected, f"Parsing {expected} failed"-[email protected]("test_type,serialized, expected", [- (Optional[str], '"foobar"', "foobar"),- (Optional[str], "null", None),- (Union[str, float], "3.14", 3.14),- (Union[str, bool], "true", True)- ])-def test_unions(test_type:type, serialized:str, expected:Any):+++ "test_type,serialized, expected",+ [+ (Optional[str], '"foobar"', "foobar"),+ (Optional[str], "null", None),+ (Union[str, float], "3.14", 3.14),+ (Union[str, bool], "true", True),+ ],+)+def test_unions(test_type: type, serialized: str, expected: Any):class MySignature(dspy.Signature):question: str = dspy.InputField()answer: test_type = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)-+assert parser(serialized) == expected, f"Parsing {expected} failed"--++def test_pydantic():class MySignature(dspy.Signature):question: str = dspy.InputField()answer: MyModel = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)-+instance = build_model_instance()parsed_instance = parser(instance.model_dump_json())-+assert parsed_instance == instance, f"{instance} != {parsed_instance}"--def test_optional_pydantic():+++def test_optional_pydantic():class MySignature(dspy.Signature):question: str = dspy.InputField()answer: Optional[MyModel] = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)-+instance = build_model_instance()parsed_instance = parser(instance.model_dump_json())assert parsed_instance == instance, f"{instance} != {parsed_instance}"-- #Check null case++ # Check null caseparsed_instance = parser("null")- assert parsed_instance == None, "Optional[MyModel] should be None"-+ assert parsed_instance == None, "Optional[MyModel] should be None"++def test_dataclass():-from dataclasses import dataclass-- @dataclass(frozen = True)++ @dataclass(frozen=True)class MyDataclass:string: strnumber: intfloating: floatboolean: bool-+class MySignature(dspy.Signature):question: str = dspy.InputField()answer: MyDataclass = dspy.OutputField()-+_, parser = get_field_and_parser(MySignature)-+instance = MyDataclass("foobar", 42, 3.14, True)parsed_instance = parser('{"string": "foobar", "number": 42, "floating": 3.14, "boolean": true}')- assert parsed_instance == instance, f"{instance} != {parsed_instance}"\ No newline at end of file+ assert parsed_instance == instance, f"{instance} != {parsed_instance}"