DSPy: The framework for programming—not prompting—language models
Python
in-progress retry up to generate
Arnav Singhvi4f862887c775
diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.pyindex 4943856b9..52d3ffa82 100644--- a/dspy/predict/chain_of_thought.py+++ b/dspy/predict/chain_of_thought.py@@ -47,11 +47,14 @@ class ChainOfThought(Predict):self.extended_signature = dsp.Template(signature.instructions, **extended_kwargs)def forward(self, **kwargs):- signature = self.signature-- if self.activated is True or (self.activated is None and isinstance(dsp.settings.lm, dsp.GPT3)):- signature = self.extended_signature-+ signature_kwargs = kwargs.pop('signature', None)+ if signature_kwargs is None:+ if self.activated is True or (self.activated is None and isinstance(dsp.settings.lm, dsp.GPT3)):+ signature = self.extended_signature+ else:+ signature = self.signature+ else:+ signature = dsp.Template(self.signature.instructions, **signature_kwargs)return super().forward(signature=signature, **kwargs)diff --git a/dspy/predict/retry.py b/dspy/predict/retry.pyindex 675313e43..55a0ad8f8 100644--- a/dspy/predict/retry.py+++ b/dspy/predict/retry.py@@ -13,15 +13,17 @@ class Retry(Predict):def _create_new_signature(self, original_signature):extended_signature = {}-- # FIXME: AttributeError: '_SignatureNamespace' object- # has no attribute 'items'.- # FIXME: the order of the input fields might be incorrect- # the new fields should be after the old input fields,- # but before the output fields.- for key, value in original_signature.items():- extended_signature[key] = value-+ input_fields = original_signature.input_fields()+ output_fields = original_signature.output_fields()+ modified_output_fields = {}+ for key, value in output_fields.items():+ modified_output_fields[f"past_{key}"] = dspy.OutputField(+ prefix='Past ' + value.prefix,+ desc=value.desc,+ format=value.format+ )+ extended_signature.update(input_fields)+ extended_signature.update(modified_output_fields)extended_signature["traces"] = dspy.InputField(prefix="Trace:",desc="Traces from your past attempts",@@ -31,7 +33,9 @@ class Retry(Predict):extended_signature["feedback"] = dspy.InputField(prefix="Instruction:", desc="Some instructions you must satisfy", format=str)+ extended_signature.update(output_fields)return extended_signature- def forward(self, **kwargs):- return self.module.forward(signature=self.new_signature, **kwargs)+ def forward(self, *args, **kwargs):+ kwargs['signature'] = self.new_signature+ return self.module.forward(**kwargs)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex f10894956..86e4689fe 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -2,10 +2,8 @@ import inspectfrom typing import Any, Callableimport dspimport dspy-import loggingimport uuid-from ..predict.retry import Retry#################### Assertion Helpers ####################@@ -213,49 +211,41 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):with updated signature if a suggestion fails. updated signature adds a newinput field to the signature, which is the feedback."""+ from ..predict.retry import Retrydef wrapper(*args, **kwargs):+ nonlocal functarget_module = handler_args.get("target_module", None)backtrack_to = Noneerror_msg = Noneresult = None- extended_predictors_to_original_forward = {}++ predictors_to_original_forward = {}# predictor_feedback: Predictor -> List[feedback_msg]predictor_feedbacks = {}- # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]+ # failure_traces: Predictor -> Dict[assertion, (ip, op, msg)]failure_traces = {}for i in range(max_backtracks + 1):if i > 0 and backtrack_to is not None:-<<<<<<< HEAD- retry_module = Retry(backtrack_to)-- # set feedback field for predictor's forward function-=======# create a new retry module that wraps backtrack_to- retry_module = dspy.Retry(backtrack_to)+ retry_module = Retry(backtrack_to)# save the original forward function to revert back topredictors_to_original_forward[backtrack_to] = backtrack_to.forward# set the new fields->>>>>>> 759d62ae604e5924cd2df6ffeb94a0d692939488+ #TODO some handling of the past_outputs fields within Retry module?feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])-- # set traces field for predictor's forward functiontrace_passages = _build_trace_passages(failure_traces[backtrack_to])kwargs["feedback"] = feedback_msgkwargs["traces"] = trace_passages-<<<<<<< HEAD- kwargs['feedback'] = feedback_msg- kwargs['traces'] = trace_passages-=======- # FIXME: need to replace backtrack_to.forward- # in the user's program with retry_module.forward+ # TODO: verify if this correctly - replace backtrack_to.forward in the user's program with retry_module.forward+ func = retry_module.forward.__get__(backtrack_to, backtrack_to.__class__)+ # setattr(backtrack_to, 'forward', func)# NOTE: this replacement needs to be thread safe if possible->>>>>>> 759d62ae604e5924cd2df6ffeb94a0d692939488# if last backtrack: ignore suggestion errorsif i == max_backtracks:@@ -264,12 +254,10 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):else:try:- result = retry_module(*args, **kwargs)+ result = func(*args, **kwargs)breakexcept DSPySuggestionError as e:- suggest_id, error_msg = e.id, e.msg- error_ip = e.state[-1][1]- error_op = e.state[-1][2].__dict__["_store"]+ suggest_id, error_msg, error_state = e.id, e.msg, e.state[-1]if dsp.settings.trace:if target_module:@@ -292,6 +280,11 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):predictor_feedbacks[backtrack_to].append(error_msg)# save latest failure trace for predictor per suggestion+ error_ip = error_state[1]+ error_op = error_state[2].__dict__["_store"]+ error_op.pop("_assert_feedback", None)+ error_op.pop("_assert_traces", None)+failure_traces.setdefault(backtrack_to, {})[suggest_id] = (error_ip,error_op,@@ -304,13 +297,10 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):)# revert any extended predictors to their originals- if extended_predictors_to_original_forward:- for (- predictor,- original_forward,- ) in extended_predictors_to_original_forward.items():+ if predictors_to_original_forward:+ for predictor, original_forward in predictors_to_original_forward.items():_revert_predictor_signature(predictor, "feedback")- predictor.forward = original_forward+ setattr(predictor, 'forward', original_forward.__get__(predictor))return result