Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Made functional more robust

Thomas D Ahleec731f7ee040
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index 15cf41a27..3003e9eef 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -46,7 +46,7 @@ class FunctionalModule(dspy.Module):
self.__dict__[name] = attr.copy()
-def TypedChainOfThought(signature, make_example=False):
+def TypedChainOfThought(signature):
""" Just like TypedPredictor, but adds a ChainOfThought OutputField. """
signature = ensure_signature(signature)
output_keys = ", ".join(signature.output_fields.keys())
@@ -56,18 +56,17 @@ def TypedChainOfThought(signature, make_example=False):
prefix="Reasoning: Let's think step by step in order to",
desc="${produce the " + output_keys + "}. We ...",
),
- ), make_example)
+ ))
class TypedPredictor(dspy.Module):
- def __init__(self, signature, make_example=False):
+ def __init__(self, signature):
super().__init__()
self.signature = signature
self.predictor = dspy.Predict(signature)
- self.make_example = make_example
def copy(self):
- return TypedPredictor(self.signature, self.make_example)
+ return TypedPredictor(self.signature)
@staticmethod
def _make_example(type_):
@@ -120,7 +119,6 @@ class TypedPredictor(dspy.Module):
+ (
f". Respond with a single JSON object using the schema "
+ json.dumps(type_.model_json_schema())
- + (". For example: " + self._make_example(type_) if self.make_example else "")
),
format=lambda x: (
x if isinstance(x, str) else x.model_dump_json()
@@ -155,11 +153,23 @@ class TypedPredictor(dspy.Module):
parser = field.json_schema_extra.get("parser", lambda x: x)
parsed_results[name] = parser(value)
except (pydantic.ValidationError, ValueError) as e:
- errors[name] = e
+ errors[name] = _format_error(e)
+ # If we can, we add an example to the error message
+ current_desc = field.json_schema_extra.get("desc", "")
+ i = current_desc.find("Respond with a single JSON object using the schema")
+ if i == -1:
+ continue # Only add examples to JSON objects
+ suffix, current_desc = current_desc[i:], current_desc[:i]
+ prefix = "You MUST use this format: "
+ if try_i + 1 < MAX_RETRIES and prefix not in current_desc:
+ if example := self._make_example(field.annotation):
+ signature = signature.with_updated_fields(name,
+ desc = current_desc + "\n" + prefix + example + "\n" + suffix
+ )
if errors:
# Add new fields for each error
for name, error in errors.items():
- modified_kwargs[f"error_{name}_{try_i}"] = str(error)
+ modified_kwargs[f"error_{name}_{try_i}"] = error
signature = signature.append(
f"error_{name}_{try_i}",
dspy.InputField(
@@ -173,7 +183,19 @@ class TypedPredictor(dspy.Module):
for name, value in parsed_results.items():
setattr(result, name, value)
return result
- raise ValueError("Too many retries")
+ raise ValueError("Too many retries trying to get the correct output format. "
+ + "Try simplifying the requirements.")
+
+
+def _format_error(error: Exception):
+ if isinstance(error, pydantic.ValidationError):
+ errors = []
+ for e in error.errors():
+ fields = ", ".join(e["loc"])
+ errors.append(f"{e['msg']}: {fields} (error type: {e['type']})")
+ return "; ".join(errors)
+ else:
+ return str(error)
def _func_to_signature(func):
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 438891a55..216ec3315 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -3,9 +3,7 @@ import textwrap
import pydantic
from pydantic import Field, BaseModel, field_validator
from typing import Annotated
-import warnings
from typing import List
-from pyparsing import Literal
import pytest
@@ -118,6 +116,7 @@ def test_simple_class():
"What is the speed of light?",
"Some bad reasoning, 3e8 m/s.",
"3e8", # Bad answer 1
+ "{...}", # Model is asked to create an example
"Some good reasoning...",
expected.model_dump_json(), # Good answer
]
@@ -295,6 +294,8 @@ def test_regex():
[
# Example with a bad origin code.
'{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
+ # Example to help the model understand
+ '{...}',
# Fixed
'{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
]
@@ -343,7 +344,9 @@ def test_multi_errors():
[
# First origin is wrong, then destination, then all is good
'{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
+ '{...}', # Example to help the model understand
'{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}',
+ '{...}', # Example to help the model understand
'{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
]
)
@@ -352,7 +355,6 @@ def test_multi_errors():
assert flight_information(email="Some email") == TravelInformation(
origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25)
)
- warnings.warn("This test is dependent on the version of pydantic used.")
assert lm.get_convo(-1) == textwrap.dedent(
"""\
Given the fields `email`, produce the fields `flight_information`.
@@ -373,12 +375,11 @@ def test_multi_errors():
Email: Some email
- Past Error (flight_information): 1 validation error for TravelInformation origin String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='JF0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch
+ Past Error (flight_information): String should match pattern '^[A-Z]{3}$': origin (error type: string_pattern_mismatch)
- Past Error (flight_information, 2): 1 validation error for TravelInformation destination String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='LA0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch
+ Past Error (flight_information, 2): String should match pattern '^[A-Z]{3}$': destination (error type: string_pattern_mismatch)
Flight Information: {"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}"""
- # Note: Pydantic version is hardcoded in the url here
)
@@ -411,7 +412,6 @@ def test_field_validator():
with pytest.raises(ValueError):
get_user_details()
- warnings.warn("This test is dependent on the version of pydantic used.")
assert lm.get_convo(-1) == textwrap.dedent(
"""\
Given the fields , produce the fields `get_user_details`.
@@ -426,7 +426,7 @@ def test_field_validator():
---
- Past Error (get_user_details): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error
- Past Error (get_user_details, 2): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error
+ Past Error (get_user_details): Value error, Name must be in uppercase.: name (error type: value_error)
+ Past Error (get_user_details, 2): Value error, Name must be in uppercase.: name (error type: value_error)
Get User Details: {"name": "lower case name", "age": 25}"""
)