DSPy: The framework for programming—not prompting—language models
Python
backtrack w/ feedback [draft]
manishshettyma18446794622
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex ab669e96d..fed1a04e5 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -47,7 +47,7 @@ class Assert:def assert_transform(backtrack=2):"""Decorator that simply re-runs the function if assertion fails,- up to [backtrack] times."""+ up to `backtrack` times."""def wrapper(func):def inner(*args, **kwargs):@@ -57,7 +57,7 @@ def assert_transform(backtrack=2):except DSPyAssertionError as e:print(f"{e.msg}")if dsp.settings.trace:- print(dsp.settings.trace[-1])+ passelse:print("UNREACHABLE: No trace available, this should not happen. Is this run time?"@@ -70,7 +70,7 @@ def assert_transform(backtrack=2):def assert_update_transform(backtrack=2):"""Decorator that simply re-runs the function with updated temperature- if assertion fails, up to [backtrack] times."""+ if assertion fails, up to `backtrack` times."""def wrapper(func):def inner(*args, **kwargs):@@ -78,7 +78,7 @@ def assert_update_transform(backtrack=2):lm = dsp.settings.lmif i > 0 and backtrack_to is not None:- print(f"updating temperature to {0.71 + 0.002 * i}")+ # print(f"updating temperature to {0.71 + 0.002 * i}")lm = lm.copy(temperature=0.71 + 0.002 * i)new_settings = dict(lm=lm) if i > 0 else {}@@ -101,9 +101,9 @@ def assert_update_transform(backtrack=2):return wrapper-def assert_latest_transform(backtrack=3):+def assert_latest_transform(backtrack=2):"""Decorator that simply re-runs the function but updates the temperature- only for the latest predictor in ther trace if assertion fails, up to [backtrack] times.+ only for the latest predictor in the trace if assertion fails, up to `backtrack` times.# NOTE (@manish): the previous assert_update_transform applies temperature change to all predictors in the pipeline.# Here, we udpate/pass the new temperature to the *backtrack_to* predictor *only* --> cause a cache miss --> re-run@@ -118,23 +118,89 @@ def assert_latest_transform(backtrack=3):if i > 0 and backtrack_to is not None:print(f"rewinding to {id(backtrack_to)}")- # udpate temperature via "config" attribute of the backtrack_to predictor-+ # udpate temperature via "config" attribute of the `backtrack_to` predictor+ predictor_id = id(backtrack_to)print(- f"prev temp @ {id(backtrack_to)}: {backtrack_to.get_config().get('temperature', 'default')}"+ f"prev temp @ {predictor_id}: {backtrack_to.get_config().get('temperature', 'default')}")backtrack_to.update_config(temperature=0.7 + 0.001 * i)print(- f"new temp @ {id(backtrack_to)}: {backtrack_to.get_config()['temperature']}"+ f"new temp @ {predictor_id}: {backtrack_to.get_config()['temperature']}")+ try:+ return func(*args, **kwargs)+ except DSPyAssertionError as e:+ print(f"{e.msg}")++ if dsp.settings.trace:+ backtrack_to = dsp.settings.trace[-1][0]+ else:+ print(+ "UNREACHABLE: No trace available, this should not happen. Is this run time?"+ )++ return inner++ return wrapper+- # @manish: works quite well w/ current example!!!- # TODO (@manish): check if this makes sense w/ more examples+def assert_latest_feedback_transform(backtrack=2):+ """"Decorator that simply re-runs the function but updates the signature of+ the latest predictor in the trace if assertion fails, up to `backtrack` times.++ The updated signature passes a new input to the predictor, which is the feedback+ from the failed assertion. This feedback will be used by the predictor to self-repair.+ """++ def wrapper(func):+ def inner(*args, **kwargs):+ backtrack_to = None+ error_msg = None++ def _extend_predictor_signature(predictor, **kwargs):+ """Update the signature of a predictor instance with a new field.+ """+ old_signature = predictor.extended_signature+ *keys, last_key = old_signature.kwargs.keys()++ extended_kwargs = {key: kwargs[key] for key in kwargs}+ extended_kwargs.update({key: old_signature.kwargs[key] for key in keys})+ extended_kwargs.update({last_key: old_signature.kwargs[last_key]})++ new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)+ predictor.extended_signature = new_signature+++ def _wrap_forward_with_set_fields(predictor, default_args):+ """Wrap the forward method of a predictor instance to enforce default arguments.+ """+ original_forward = predictor.forward++ def new_forward(**kwargs):+ for arg, value in default_args.items():+ kwargs.setdefault(arg, value)++ return original_forward(**kwargs)++ predictor.forward = new_forward+++ for i in range(backtrack):+ if i > 0 and backtrack_to is not None:+ print(f"rewinding to {id(backtrack_to)}")++ # udpate the signature of the `backtrack_to` predictor with a feedback field+ feedback = dspy.InputField(prefix="Note:", desc="common mistakes to avoid")+ _extend_predictor_signature(backtrack_to, feedback=feedback)++ # set the feedback field as a default kwarg to the predictor's forward function+ _wrap_forward_with_set_fields(backtrack_to, default_args={'feedback': error_msg})try:return func(*args, **kwargs)except DSPyAssertionError as e:print(f"{e.msg}")+ error_msg = "Query length should be less than 100 characters."if dsp.settings.trace:backtrack_to = dsp.settings.trace[-1][0]@@ -145,4 +211,4 @@ def assert_latest_transform(backtrack=3):return inner- return wrapper+ return wrapper\ No newline at end of filediff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex 3704b2dd1..bef53168c 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -28,7 +28,7 @@ class Module(BaseModule, metaclass=ProgramMeta):if getattr(self, "forward", False) and not getattr(self.forward, "_decorated", False):- wrapped_forward = assert_latest_feedback_transform()(self.forward)+ wrapped_forward = assert_latest_transform()(self.forward)return wrapped_forward(*args, **kwargs)else:return self.forward(*args, **kwargs)