Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added Retry module

Arnav Singhvi718ce557b1f7
diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py
index 2dec22fe5..b6a614903 100644
--- a/dspy/predict/__init__.py
+++ b/dspy/predict/__init__.py
@@ -4,4 +4,5 @@ from .multi_chain_comparison import MultiChainComparison
from .chain_of_thought_with_hint import ChainOfThoughtWithHint
from .react import ReAct
from .aggregation import majority
-from .program_of_thought import ProgramOfThought
\ No newline at end of file
+from .program_of_thought import ProgramOfThought
+from .retry import Retry
\ No newline at end of file
diff --git a/dspy/predict/retry.py b/dspy/predict/retry.py
new file mode 100644
index 000000000..0c7896718
--- /dev/null
+++ b/dspy/predict/retry.py
@@ -0,0 +1,28 @@
+import dspy
+import dsp
+
+class Retry(dspy.Predict):
+ def __init__(self, module):
+ super().__init__(module.signature)
+ self.module = module
+ self.original_signature = module.signature.signature
+ self.new_signature = self._create_new_signature(self.original_signature)
+
+ def _create_new_signature(self, original_signature):
+ extended_signature = {}
+ for key, value in original_signature.items():
+ extended_signature[key] = value
+ extended_signature['traces'] = dspy.InputField(
+ prefix="Trace:",
+ desc="Traces from your past attempts",
+ format=dsp.passages2text
+ )
+ extended_signature['feedback'] = dspy.InputField(
+ prefix="Instruction:",
+ desc="Some instructions you must satisfy",
+ format=str
+ )
+ return extended_signature
+
+ def forward(self, **kwargs):
+ return self.module.forward(signature=self.new_signature, **kwargs)
\ No newline at end of file
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index 0407349fe..d126749fc 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -5,7 +5,7 @@ import dspy
import logging
import uuid
-
+from ..predict.retry import Retry
#################### Assertion Helpers ####################
@@ -37,12 +37,14 @@ def _build_error_msg(feedback_msgs):
def _build_trace_passages(failure_traces):
"""Build a list of trace passages from a list of failure traces."""
- trace_template = """Failed Instruction: {} | Output: {}"""
- trace_passages = []
- for id, (ip, op, msg) in failure_traces.items():
- trace_passages.append(trace_template.format(msg, op))
- return trace_passages
+ # print("Traces:")
+ # for id, (ip, op, msg) in failure_traces.items():
+ # print(id, op, msg)
+ # print()
+
+ # TODO complete this
+ return failure_traces
def _extend_predictor_signature(predictor, **kwargs):
@@ -114,19 +116,11 @@ class DSPySuggestionError(AssertionError):
class Constraint:
- # a registry of all constraints created
- _registry = {}
-
def __init__(self, result: bool, msg: str = "", target_module=None):
- key = (result, msg, target_module)
- if key in Constraint._registry:
- self = Constraint._registry[key]
- else:
- self.id = str(uuid.uuid4())
- self.result = result
- self.msg = msg
- self.target_module = target_module
- Constraint._registry[key] = self
+ self.id = str(uuid.uuid4())
+ self.result = result
+ self.msg = msg
+ self.target_module = target_module
self.__call__()
@@ -238,51 +232,25 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
backtrack_to = None
error_msg = None
result = None
-
- predictors_to_original_forward = {}
+ extended_predictors_to_original_forward = {}
# predictor_feedback: Predictor -> List[feedback_msg]
predictor_feedbacks = {}
- # failure_traces: Predictor -> Dict[assertion, (ip, op, msg)]
+ # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]
failure_traces = {}
for i in range(max_backtracks + 1):
if i > 0 and backtrack_to is not None:
- # create a new traces field
- traces_field = dspy.InputField(
- prefix="Counter Examples:",
- desc="Traces of some incorrect outputs that violate instructions",
- format=dsp.passages2text,
- )
-
- # create a new feedback field
- feedback_field = dspy.InputField(
- prefix="Instruction:",
- desc="Some instructions you must satisfy",
- format=str,
- )
+ retry_module = Retry(backtrack_to)
- # save the original forward function to revert back to
- predictors_to_original_forward[backtrack_to] = backtrack_to.forward
-
- # extend signature with new fields
- _extend_predictor_signature(
- backtrack_to,
- _assert_feedback=feedback_field,
- _assert_traces=traces_field,
- )
-
- # set the new fields
+ # set feedback field for predictor's forward function
feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])
+
+ # set traces field for predictor's forward function
trace_passages = _build_trace_passages(failure_traces[backtrack_to])
- _wrap_forward_with_set_fields(
- backtrack_to,
- default_args={
- "_assert_feedback": feedback_msg,
- "_assert_traces": trace_passages,
- },
- )
+ kwargs['feedback'] = feedback_msg
+ kwargs['traces'] = trace_passages
# if last backtrack: ignore suggestion errors
if i == max_backtracks:
@@ -291,10 +259,12 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
else:
try:
- result = func(*args, **kwargs)
+ result = retry_module(*args, **kwargs)
break
except DSPySuggestionError as e:
- suggest_id, error_msg, error_state = e.id, e.msg, e.state[-1]
+ suggest_id, error_msg = e.id, e.msg
+ error_ip = e.state[-1][1]
+ error_op = e.state[-1][2].__dict__["_store"]
if dsp.settings.trace:
if target_module:
@@ -317,11 +287,6 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
predictor_feedbacks[backtrack_to].append(error_msg)
# save latest failure trace for predictor per suggestion
- error_ip = error_state[1]
- error_op = error_state[2].__dict__["_store"]
- error_op.pop("_assert_feedback", None)
- error_op.pop("_assert_traces", None)
-
failure_traces.setdefault(backtrack_to, {})[suggest_id] = (
error_ip,
error_op,
@@ -334,8 +299,11 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
)
# revert any extended predictors to their originals
- if predictors_to_original_forward:
- for predictor, original_forward in predictors_to_original_forward.items():
+ if extended_predictors_to_original_forward:
+ for (
+ predictor,
+ original_forward,
+ ) in extended_predictors_to_original_forward.items():
_revert_predictor_signature(predictor, "feedback")
predictor.forward = original_forward
@@ -344,9 +312,6 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
return wrapper
-#################### Assertion Module Transformer ####################
-
-
def handle_assert_forward(assertion_handler, **handler_args):
def forward(self, *args, **kwargs):
args_to_vals = inspect.getcallargs(self._forward, *args, **kwargs)
@@ -385,4 +350,4 @@ def assert_transform_module(
module
)
- return module
+ return module
\ No newline at end of file
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index 5a92964ae..ea31a46d4 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -12,6 +12,13 @@ class SignatureMeta(type):
for key, value in fields.items():
setattr(self, key, value)
+ def input_fields(self):
+ return {k: v for k, v in self.__dict__.items() if isinstance(v, InputField)}
+
+ def output_fields(self):
+ return {k: v for k, v in self.__dict__.items() if isinstance(v, OutputField)}
+
+
def __new__(cls, name, bases, class_dict):
type_attributes = {}
@@ -48,7 +55,7 @@ class SignatureMeta(type):
# Redirect attribute access to the template object when accessed on the class directly
if attr not in cls.__dict__:
return getattr(cls._template, attr)
- return super().__getattr__(attr)
+ return super().__getattr__(attr)
class Signature(metaclass=SignatureMeta):
def __init__(self, signature: str = "", instructions: str = ""):
@@ -166,4 +173,4 @@ assert infer_prefix('someAttributeName') == 'Some Attribute Name'
assert infer_prefix('some_attribute_name') == 'Some Attribute Name'
assert infer_prefix('URLAddress') == 'URL Address'
assert infer_prefix('isHTTPSecure') == 'Is HTTP Secure'
-assert infer_prefix('isHTTPSSecure123') == 'Is HTTPS Secure 123'
+assert infer_prefix('isHTTPSSecure123') == 'Is HTTPS Secure 123'
\ No newline at end of file