Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

backtrack w/ feedback [draft]

manishshettyma18446794622
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index ab669e96d..fed1a04e5 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -47,7 +47,7 @@ class Assert:
def assert_transform(backtrack=2):
"""Decorator that simply re-runs the function if assertion fails,
- up to [backtrack] times."""
+ up to `backtrack` times."""
def wrapper(func):
def inner(*args, **kwargs):
@@ -57,7 +57,7 @@ def assert_transform(backtrack=2):
except DSPyAssertionError as e:
print(f"{e.msg}")
if dsp.settings.trace:
- print(dsp.settings.trace[-1])
+ pass
else:
print(
"UNREACHABLE: No trace available, this should not happen. Is this run time?"
@@ -70,7 +70,7 @@ def assert_transform(backtrack=2):
def assert_update_transform(backtrack=2):
"""Decorator that simply re-runs the function with updated temperature
- if assertion fails, up to [backtrack] times."""
+ if assertion fails, up to `backtrack` times."""
def wrapper(func):
def inner(*args, **kwargs):
@@ -78,7 +78,7 @@ def assert_update_transform(backtrack=2):
lm = dsp.settings.lm
if i > 0 and backtrack_to is not None:
- print(f"updating temperature to {0.71 + 0.002 * i}")
+ # print(f"updating temperature to {0.71 + 0.002 * i}")
lm = lm.copy(temperature=0.71 + 0.002 * i)
new_settings = dict(lm=lm) if i > 0 else {}
@@ -101,9 +101,9 @@ def assert_update_transform(backtrack=2):
return wrapper
-def assert_latest_transform(backtrack=3):
+def assert_latest_transform(backtrack=2):
"""Decorator that simply re-runs the function but updates the temperature
- only for the latest predictor in ther trace if assertion fails, up to [backtrack] times.
+ only for the latest predictor in the trace if assertion fails, up to `backtrack` times.
# NOTE (@manish): the previous assert_update_transform applies temperature change to all predictors in the pipeline.
# Here, we udpate/pass the new temperature to the *backtrack_to* predictor *only* --> cause a cache miss --> re-run
@@ -118,23 +118,89 @@ def assert_latest_transform(backtrack=3):
if i > 0 and backtrack_to is not None:
print(f"rewinding to {id(backtrack_to)}")
- # udpate temperature via "config" attribute of the backtrack_to predictor
-
+ # udpate temperature via "config" attribute of the `backtrack_to` predictor
+ predictor_id = id(backtrack_to)
print(
- f"prev temp @ {id(backtrack_to)}: {backtrack_to.get_config().get('temperature', 'default')}"
+ f"prev temp @ {predictor_id}: {backtrack_to.get_config().get('temperature', 'default')}"
)
backtrack_to.update_config(temperature=0.7 + 0.001 * i)
print(
- f"new temp @ {id(backtrack_to)}: {backtrack_to.get_config()['temperature']}"
+ f"new temp @ {predictor_id}: {backtrack_to.get_config()['temperature']}"
)
+ try:
+ return func(*args, **kwargs)
+ except DSPyAssertionError as e:
+ print(f"{e.msg}")
+
+ if dsp.settings.trace:
+ backtrack_to = dsp.settings.trace[-1][0]
+ else:
+ print(
+ "UNREACHABLE: No trace available, this should not happen. Is this run time?"
+ )
+
+ return inner
+
+ return wrapper
+
- # @manish: works quite well w/ current example!!!
- # TODO (@manish): check if this makes sense w/ more examples
+def assert_latest_feedback_transform(backtrack=2):
+ """"Decorator that simply re-runs the function but updates the signature of
+ the latest predictor in the trace if assertion fails, up to `backtrack` times.
+
+ The updated signature passes a new input to the predictor, which is the feedback
+ from the failed assertion. This feedback will be used by the predictor to self-repair.
+ """
+
+ def wrapper(func):
+ def inner(*args, **kwargs):
+ backtrack_to = None
+ error_msg = None
+
+ def _extend_predictor_signature(predictor, **kwargs):
+ """Update the signature of a predictor instance with a new field.
+ """
+ old_signature = predictor.extended_signature
+ *keys, last_key = old_signature.kwargs.keys()
+
+ extended_kwargs = {key: kwargs[key] for key in kwargs}
+ extended_kwargs.update({key: old_signature.kwargs[key] for key in keys})
+ extended_kwargs.update({last_key: old_signature.kwargs[last_key]})
+
+ new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)
+ predictor.extended_signature = new_signature
+
+
+ def _wrap_forward_with_set_fields(predictor, default_args):
+ """Wrap the forward method of a predictor instance to enforce default arguments.
+ """
+ original_forward = predictor.forward
+
+ def new_forward(**kwargs):
+ for arg, value in default_args.items():
+ kwargs.setdefault(arg, value)
+
+ return original_forward(**kwargs)
+
+ predictor.forward = new_forward
+
+
+ for i in range(backtrack):
+ if i > 0 and backtrack_to is not None:
+ print(f"rewinding to {id(backtrack_to)}")
+
+ # udpate the signature of the `backtrack_to` predictor with a feedback field
+ feedback = dspy.InputField(prefix="Note:", desc="common mistakes to avoid")
+ _extend_predictor_signature(backtrack_to, feedback=feedback)
+
+ # set the feedback field as a default kwarg to the predictor's forward function
+ _wrap_forward_with_set_fields(backtrack_to, default_args={'feedback': error_msg})
try:
return func(*args, **kwargs)
except DSPyAssertionError as e:
print(f"{e.msg}")
+ error_msg = "Query length should be less than 100 characters."
if dsp.settings.trace:
backtrack_to = dsp.settings.trace[-1][0]
@@ -145,4 +211,4 @@ def assert_latest_transform(backtrack=3):
return inner
- return wrapper
+ return wrapper
\ No newline at end of file
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index 3704b2dd1..bef53168c 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -28,7 +28,7 @@ class Module(BaseModule, metaclass=ProgramMeta):
if getattr(self, "forward", False) and not getattr(
self.forward, "_decorated", False
):
- wrapped_forward = assert_latest_feedback_transform()(self.forward)
+ wrapped_forward = assert_latest_transform()(self.forward)
return wrapped_forward(*args, **kwargs)
else:
return self.forward(*args, **kwargs)