Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Refactor Retry: no change to module during runtime

Shangyinte42b09c1710e
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.py
index e96c8348a..4ec410fa2 100644
--- a/assert_hotpotqa.py
+++ b/assert_hotpotqa.py
@@ -1,6 +1,7 @@
import dspy
from dsp.utils import deduplicate
from dspy.datasets import HotPotQA
+from dspy.predict.retry import Retry
from dspy.teleprompt import BootstrapFewShot
# pipeline configs
@@ -90,6 +91,7 @@ def validate_query_distinction_local(previous_queries, query):
# declaration of dspy program
class SimplifiedBaleen(dspy.Module):
+
def __init__(self, passages_per_hop=2, max_hops=2):
super().__init__()
@@ -128,8 +130,11 @@ class SimplifiedBaleen(dspy.Module):
teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
+
compiled_baleen = teleprompter.compile(
- SimplifiedBaleen(), teacher=SimplifiedBaleen(passages_per_hop=2), trainset=trainset
+ student=SimplifiedBaleen().map_named_predictors(Retry),
+ teacher=SimplifiedBaleen(passages_per_hop=2).map_named_predictors(Retry),
+ trainset=trainset,
)
print("=" * 50, "Validation Failures", "=" * 50)
diff --git a/dspy/predict/retry.py b/dspy/predict/retry.py
index d8f7baabb..b0bd75caa 100644
--- a/dspy/predict/retry.py
+++ b/dspy/predict/retry.py
@@ -45,3 +45,11 @@ class Retry(Predict):
del kwargs["past_outputs"]
kwargs["signature"] = self.new_signature
return self.original_forward(**kwargs)
+
+ def __call__(self, **kwargs):
+ if dspy.settings.backtrack_to == self.module:
+ for key, value in dspy.settings.backtrack_to_args.items():
+ kwargs.setdefault(key, value)
+ return self.forward(**kwargs)
+ else:
+ return self.module(**kwargs)
\ No newline at end of file
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index da1576554..5a7db0cff 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -136,6 +136,7 @@ class DSPySuggestionError(AssertionError):
class Constraint:
+
def __init__(self, result: bool, msg: str = "", target_module=None):
self.id = str(uuid.uuid4())
self.result = result
@@ -252,49 +253,23 @@ def suggest_backtrack_handler(func, max_backtracks=2):
from ..predict.retry import Retry
def wrapper(*args, **kwargs):
- backtrack_to, error_msg, result = None, None, None
+ error_msg, result = None, None
+ dspy.settings.backtrack_to = None
# Predictor -> List[feedback_msg]
dspy.settings.predictor_feedbacks = {}
- # Predictor -> Predictor.forward
- dspy.settings.predictors_to_original_forward = {}
-
for i in range(max_backtracks + 1):
- if i > 0 and backtrack_to is not None:
- # revert to original forward function if modified
- if backtrack_to in dspy.settings.predictors_to_original_forward:
- backtrack_to.forward = dspy.settings.predictors_to_original_forward[
- backtrack_to
- ]
-
- # create a new retry module wrapping backtrack_to
- retry_module = Retry(backtrack_to)
-
- # save original forward function to revert back to
- dspy.settings.predictors_to_original_forward[
- backtrack_to
- ] = backtrack_to.forward
-
+ if i > 0 and dspy.settings.backtrack_to is not None:
# generate values for new fields
feedback_msg = _build_error_msg(
- dspy.settings.predictor_feedbacks[backtrack_to]
+ dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to]
)
- # set values as default for the new fields
- _wrap_forward_with_set_fields(
- retry_module,
- default_args={
- "feedback": feedback_msg,
- "past_outputs": past_outputs,
- },
- )
-
- # point backtrack_to.forward to retry_module.forward
- # TODO: verify and make sure is thread safe
- backtrack_to.forward = retry_module.forward.__get__(
- retry_module, retry_module.__class__
- )
+ dspy.settings.backtrack_to_args = {
+ "feedback": feedback_msg,
+ "past_outputs": past_outputs,
+ }
# if last backtrack: ignore suggestion errors
if i == max_backtracks:
@@ -319,24 +294,24 @@ def suggest_backtrack_handler(func, max_backtracks=2):
trace_element = dsp.settings.trace[i]
mod = trace_element[0]
if mod.signature == suggest_target_module:
- backtrack_to = mod
+ dspy.settings.backtrack_to = mod
break
else:
- backtrack_to = dsp.settings.trace[-1][0]
+ dspy.settings.backtrack_to = dsp.settings.trace[-1][0]
- if backtrack_to is None:
+ if dspy.settings.backtrack_to is None:
logger.error("Specified module not found in trace")
# save unique feedback message for predictor
if (
error_msg
not in dspy.settings.predictor_feedbacks.setdefault(
- backtrack_to, []
+ dspy.settings.backtrack_to, []
)
):
- dspy.settings.predictor_feedbacks[backtrack_to].append(
- error_msg
- )
+ dspy.settings.predictor_feedbacks[
+ dspy.settings.backtrack_to
+ ].append(error_msg)
output_fields = vars(error_state[0].signature.signature)
past_outputs = {}
@@ -358,13 +333,13 @@ def suggest_backtrack_handler(func, max_backtracks=2):
)
# cleanup: after all tries revert any leftover predictors to their originals
- if dspy.settings.predictors_to_original_forward:
- for (
- predictor,
- original_forward,
- ) in dspy.settings.predictors_to_original_forward.items():
- _revert_predictor_signature(predictor, "feedback")
- setattr(predictor, "forward", original_forward.__get__(predictor))
+ # if dspy.settings.predictors_to_original_forward:
+ # for (
+ # predictor,
+ # original_forward,
+ # ) in dspy.settings.predictors_to_original_forward.items():
+ # _revert_predictor_signature(predictor, "feedback")
+ # setattr(predictor, "forward", original_forward.__get__(predictor))
return result
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index 5f12aa433..5567ea5ad 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -3,6 +3,7 @@ import inspect
from dspy.primitives.module import BaseModule
from dspy.primitives.assertions import *
+import re
class ProgramMeta(type):
@@ -48,6 +49,12 @@ class Module(BaseModule, metaclass=ProgramMeta):
return "\n".join(s)
+ def map_named_predictors(self, func):
+ """Applies a function to all named predictors."""
+ for name, predictor in self.named_predictors():
+ set_attribute_by_name(self, name, func(predictor))
+ return self
+
# def __deepcopy__(self, memo):
# # memo is a dict of id's to copies already made during the current call
# # Check if the object is already copied
@@ -67,4 +74,37 @@ class Module(BaseModule, metaclass=ProgramMeta):
# return new_copy
+# FIXME(Shangyint): This may cause some problems for nested patterns.
+def set_attribute_by_name(obj, name, value):
+ # Regular expressions for different patterns
+ module_pattern = re.compile(r"^([^.]+)\.(.+)$")
+ list_pattern = re.compile(r"^([^\[]+)\[([0-9]+)\]$")
+ dict_pattern = re.compile(r"^([^\[]+)\['([^']+)'\]$")
+
+ # Match for module.attribute pattern
+ module_match = module_pattern.match(name)
+ if module_match:
+ module_name, sub_name = module_match.groups()
+ sub_obj = getattr(obj, module_name)
+ set_attribute_by_name(sub_obj, sub_name, value)
+ return
+
+ # Match for list[index] pattern
+ list_match = list_pattern.match(name)
+ if list_match:
+ list_name, index = list_match.groups()
+ getattr(obj, list_name)[int(index)] = value
+ return
+
+ # Match for dict['key'] pattern
+ dict_match = dict_pattern.match(name)
+ if dict_match:
+ dict_name, key = dict_match.groups()
+ getattr(obj, dict_name)[key] = value
+ return
+
+ # Default case for simple attributes
+ setattr(obj, name, value)
+
+
Program = Module