Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

[wip] tracking failure traces and using it for backtrack

manishshettym76715411d7bf
diff --git a/.gitignore b/.gitignore
index 3abff4193..f048c4639 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,3 +40,4 @@ finetuning_ckpts/
# cache/
* copy*
.idea
+assertion.log
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index ee4b6e2b1..52ae84a95 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -4,6 +4,10 @@ import dsp
import dspy
import logging
+import uuid
+
+
+#################### Assertion Helpers ####################
def setup_logger():
@@ -26,11 +30,74 @@ def setup_logger():
logger = setup_logger()
+def _build_error_msg(feedback_msgs):
+ """Build an error message from a list of feedback messages."""
+ return "\n".join([msg for msg in feedback_msgs])
+
+
+def _build_trace_passages(failure_traces):
+ """Build a list of trace passages from a list of failure traces."""
+
+ # print("Traces:")
+ # for id, (ip, op, msg) in failure_traces.items():
+ # print(id, op, msg)
+ # print()
+
+ # TODO complete this
+ return failure_traces
+
+
+def _extend_predictor_signature(predictor, **kwargs):
+ """Update the signature of a predictor instance with specified fields."""
+ old_signature = predictor.extended_signature
+ old_keys = list(old_signature.kwargs.keys())
+
+ # include other input fields after question
+ position = old_keys.index("question") + 1
+ for key in reversed(kwargs):
+ old_keys.insert(position, key)
+
+ extended_kwargs = {
+ key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys
+ }
+
+ new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)
+ predictor.extended_signature = new_signature
+
+
+def _revert_predictor_signature(predictor, *args):
+ """Revert the signature of a predictor by removing specified fields."""
+ old_signature_kwargs = predictor.extended_signature.kwargs
+ for key in args:
+ old_signature_kwargs.pop(key, None)
+ new_signature = dsp.Template(
+ predictor.extended_signature.instructions, **old_signature_kwargs
+ )
+ predictor.extended_signature = new_signature
+
+
+def _wrap_forward_with_set_fields(predictor, default_args):
+ """Wrap the forward method of a predictor instance to enforce default arguments."""
+ original_forward = predictor.forward
+
+ def new_forward(**kwargs):
+ for arg, value in default_args.items():
+ kwargs.setdefault(arg, value)
+
+ return original_forward(**kwargs)
+
+ predictor.forward = new_forward
+
+
+#################### Assertion Exceptions ####################
+
+
class DSPyAssertionError(AssertionError):
"""Custom exception raised when a DSPy `Assert` fails."""
- def __init__(self, msg: str, state: Any = None) -> None:
+ def __init__(self, id: str, msg: str, state: Any = None) -> None:
super().__init__(msg)
+ self.id = id
self.msg = msg
self.state = state
@@ -38,15 +105,19 @@ class DSPyAssertionError(AssertionError):
class DSPySuggestionError(AssertionError):
"""Custom exception raised when a DSPy `Suggest` fails."""
- def __init__(self, msg: str, state: Any = None) -> None:
+ def __init__(self, id: str, msg: str, state: Any = None) -> None:
super().__init__(msg)
+ self.id = id
self.msg = msg
self.state = state
-class Constraint:
+#################### Assertion Primitives ####################
+
+class Constraint:
def __init__(self, result: bool, msg: str = "", target_module=None):
+ self.id = str(uuid.uuid4())
self.result = result
self.msg = msg
self.target_module = target_module
@@ -66,7 +137,9 @@ class Assert(Constraint):
return True
else:
logger.error(f"AssertionError: {self.msg}")
- raise DSPyAssertionError(msg=self.msg, state=dsp.settings.trace)
+ raise DSPyAssertionError(
+ id=self.id, msg=self.msg, state=dsp.settings.trace
+ )
else:
raise ValueError("Assertion function should always return [bool]")
@@ -83,11 +156,16 @@ class Suggest(Constraint):
return True
else:
logger.error(f"SuggestionFailed: {self.msg}")
- raise DSPySuggestionError(msg=self.msg, state=dsp.settings.trace)
+ raise DSPySuggestionError(
+ id=self.id, msg=self.msg, state=dsp.settings.trace
+ )
else:
raise ValueError("Suggestion function should always return [bool]")
+#################### Assertion Handlers ####################
+
+
def noop_handler(func):
"""Handler to bypass assertions and suggestions.
@@ -111,7 +189,7 @@ def bypass_suggest_handler(func):
def wrapper(*args, **kwargs):
with dspy.settings.context(bypass_suggest=True, bypass_assert=False):
return func(*args, **kwargs)
-
+
return wrapper
@@ -125,7 +203,7 @@ def bypass_assert_handler(func):
def wrapper(*args, **kwargs):
with dspy.settings.context(bypass_assert=True):
return func(*args, **kwargs)
-
+
return wrapper
@@ -137,7 +215,7 @@ def assert_no_except_handler(func):
return func(*args, **kwargs)
except DSPyAssertionError as e:
return None
-
+
return wrapper
@@ -150,62 +228,26 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
"""
def wrapper(*args, **kwargs):
- target_module = handler_args.get('target_module', None)
+ target_module = handler_args.get("target_module", None)
backtrack_to = None
error_msg = None
result = None
extended_predictors_to_original_forward = {}
# predictor_feedback: Predictor -> List[feedback_msg]
- # this will contain all assertion failure msgs for each predictor
- # including msgs for the same assertion failure
predictor_feedbacks = {}
- def _extend_predictor_signature(predictor, **kwargs):
- """Update the signature of a predictor instance with specified fields."""
- old_signature = predictor.extended_signature
- old_keys = list(old_signature.kwargs.keys())
-
- # include other input fields after question
- position = old_keys.index("question") + 1
- for key in reversed(kwargs):
- old_keys.insert(position, key)
-
- extended_kwargs = {
- key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys
- }
-
- new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)
- predictor.extended_signature = new_signature
-
- def _build_error_msg(feedback_msgs):
- """Build an error message from a list of feedback messages."""
- return "\n".join([msg for msg in feedback_msgs])
-
- def _revert_predictor_signature(predictor, *args):
- """Revert the signature of a predictor by removing specified fields."""
- old_signature_kwargs = predictor.extended_signature.kwargs
- for key in args:
- old_signature_kwargs.pop(key, None)
- new_signature = dsp.Template(
- predictor.extended_signature.instructions, **old_signature_kwargs
- )
- predictor.extended_signature = new_signature
-
- def _wrap_forward_with_set_fields(predictor, default_args):
- """Wrap the forward method of a predictor instance to enforce default arguments."""
- original_forward = predictor.forward
-
- def new_forward(**kwargs):
- for arg, value in default_args.items():
- kwargs.setdefault(arg, value)
-
- return original_forward(**kwargs)
-
- predictor.forward = new_forward
+ # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]
+ failure_traces = {}
for i in range(max_backtracks + 1):
if i > 0 and backtrack_to is not None:
- # TODO clean up this function
+ # create a new traces field
+ traces = dspy.InputField(
+ prefix="Trace:",
+ desc="Traces from your past attempts",
+ format=dsp.passages2text,
+ )
+
# create a new feedback field
feedback = dspy.InputField(
prefix="Instruction:",
@@ -221,8 +263,13 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
# extend signature with feedback field
_extend_predictor_signature(backtrack_to, feedback=feedback)
- # set feedback field as a default kwarg to the predictor's forward function
+ # set feedback field for predictor's forward function
feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])
+
+ # set traces field for predictor's forward function
+ trace_passages = _build_trace_passages(failure_traces[backtrack_to])
+ # TODO: set this in the predictor's forward function
+
_wrap_forward_with_set_fields(
backtrack_to, default_args={"feedback": feedback_msg}
)
@@ -236,9 +283,10 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
try:
result = func(*args, **kwargs)
break
- # TODO Change this to suggest instead of assert
except DSPySuggestionError as e:
- error_msg = e.msg
+ suggest_id, error_msg = e.id, e.msg
+ error_ip = e.state[-1][1]
+ error_op = e.state[-1][2].__dict__["_store"]
if dsp.settings.trace:
if target_module:
@@ -250,10 +298,23 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
break
else:
backtrack_to = dsp.settings.trace[-1][0]
- if error_msg not in predictor_feedbacks.setdefault(backtrack_to, []):
- predictor_feedbacks[backtrack_to].append(error_msg)
+
if backtrack_to is None:
logger.error("Specified module not found in trace")
+
+ # save unique feedback message for predictor
+ if error_msg not in predictor_feedbacks.setdefault(
+ backtrack_to, []
+ ):
+ predictor_feedbacks[backtrack_to].append(error_msg)
+
+ # save latest failure trace for predictor per suggestion
+ failure_traces.setdefault(backtrack_to, {})[suggest_id] = (
+ error_ip,
+ error_op,
+ error_msg,
+ )
+
else:
logger.error(
f"UNREACHABLE: No trace available, this should not happen. Is this run time?"
@@ -290,7 +351,9 @@ def handle_assert_forward(assertion_handler, **handler_args):
default_assertion_handler = suggest_backtrack_handler
-def assert_transform_module(module, assertion_handler=default_assertion_handler, **handler_args):
+def assert_transform_module(
+ module, assertion_handler=default_assertion_handler, **handler_args
+):
"""
Transform a module to handle assertions.
"""
@@ -305,6 +368,8 @@ def assert_transform_module(module, assertion_handler=default_assertion_handler,
pass # TODO warning: might be overwriting a previous _forward method
module._forward = module.forward
- module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(module)
+ module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(
+ module
+ )
return module