DSPy: The framework for programming—not prompting—language models
Python
Simplify built-in modules (remove new_* & extended_signature). Prepare for assertions v2. (#1943)
Omar Khattabae86009366c6
diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.pyindex 0260ead37..e273a3dc9 100644--- a/dspy/predict/__init__.py+++ b/dspy/predict/__init__.py@@ -6,5 +6,5 @@ from .multi_chain_comparison import MultiChainComparisonfrom .predict import Predictfrom .program_of_thought import ProgramOfThoughtfrom .react import ReAct, Tool-from .retry import Retry-from .parallel import Parallel\ No newline at end of file+from .parallel import Parallel+# from .retry import Retry\ No newline at end of filediff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.pyindex b6b9cb575..4a56c0ceb 100644--- a/dspy/predict/chain_of_thought.py+++ b/dspy/predict/chain_of_thought.py@@ -2,58 +2,19 @@ import dspyfrom dspy.primitives.program import Modulefrom dspy.signatures.signature import ensure_signature-# TODO: This shouldn't inherit from Predict. It should be a module that has one or two predictors.-# Let's focus on the activated case. It's a predictor with the expanded signature.-# Now, when deactivated, it's a predictor with the original signature.-# When activate is None, though, we need the expanded one but during forward we need to pass the right signature.-class ChainOfThought(Module):- def __init__(self, signature, rationale_type=None, activated=True, **config):+ def __init__(self, signature, rationale_type=None, **config):super().__init__()- self.activated = activated-- self.signature = signature = ensure_signature(signature)- *_keys, last_key = signature.output_fields.keys()+ signature = ensure_signature(signature)prefix = "Reasoning: Let's think step by step in order to"-- if isinstance(dspy.settings.lm, dspy.LM):- desc = "${reasoning}"- elif dspy.settings.experimental:- desc = "${produce the output fields}. We ..."- else:- desc = f"${{produce the {last_key}}}. We ..."-+ desc = "${reasoning}"rationale_type = rationale_type or dspy.OutputField(prefix=prefix, desc=desc)-- # Add "rationale" field to the output signature.- if isinstance(dspy.settings.lm, dspy.LM) or dspy.settings.experimental:- extended_signature = signature.prepend("reasoning", rationale_type, type_=str)- else:- extended_signature = signature.prepend("rationale", rationale_type, type_=str)+ extended_signature = signature.prepend("reasoning", rationale_type, type_=str)- self._predict = dspy.Predict(extended_signature, **config)- self._predict.extended_signature = extended_signature+ self.predict = dspy.Predict(extended_signature, **config)def forward(self, **kwargs):- assert self.activated in [True, False]-- signature = kwargs.pop("new_signature", self._predict.extended_signature if self.activated else self.signature)- return self._predict(signature=signature, **kwargs)-- @property- def demos(self):- return self._predict.demos-- @property- def extended_signature(self):- return self._predict.extended_signature---"""-TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.--IF the user didn't overwrite our default rationale_type.-"""+ return self.predict(**kwargs)diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 3f851e7d6..05d66bb48 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -51,9 +51,6 @@ class Predict(Module, Parameter):state["demos"].append(demo)state["signature"] = self.signature.dump_state()- # `extended_signature` is a special field for `Predict`s like CoT.- if hasattr(self, "extended_signature"):- state["extended_signature"] = self.extended_signature.dump_state()return statedef load_state(self, state):@@ -82,8 +79,8 @@ class Predict(Module, Parameter):self.signature = self.signature.load_state(state["signature"])- if "extended_signature" in state:- self.extended_signature = self.extended_signature.load_state(state["extended_signature"])+ if "extended_signature" in state: # legacy, up to and including 2.5, for CoT.+ self.signature = self.signature.load_state(state["extended_signature"])return self@@ -96,14 +93,14 @@ class Predict(Module, Parameter):assert not dspy.settings.compiling, "It's no longer ever the case that .compiling is True"# Extract the three privileged keyword arguments.- new_signature = ensure_signature(kwargs.pop("new_signature", None))+ assert "new_signature" not in kwargs, "new_signature is no longer a valid keyword argument."signature = ensure_signature(kwargs.pop("signature", self.signature))demos = kwargs.pop("demos", self.demos)config = dict(**self.config, **kwargs.pop("config", {}))# Get the right LM to use.lm = kwargs.pop("lm", self.lm) or dspy.settings.lm- assert lm is not None, "No LM is loaded."+ assert isinstance(lm, dspy.LM), "No LM is loaded."# If temperature is 0.0 but its n > 1, set temperature to 0.7.temperature = config.get("temperature")@@ -113,15 +110,11 @@ class Predict(Module, Parameter):if (temperature is None or temperature <= 0.15) and num_generations > 1:config["temperature"] = 0.7- if new_signature is not None:- signature = new_signature-if not all(k in kwargs for k in signature.input_fields):present = [k for k in signature.input_fields if k in kwargs]missing = [k for k in signature.input_fields if k not in kwargs]print(f"WARNING: Not all input fields were provided to module. Present: {present}. Missing: {missing}.")- assert isinstance(lm, dspy.LM)completions = v2_5_generate(lm, config, signature, demos, kwargs, _parse_values=self._parse_values)pred = Prediction.from_completions(completions, signature=signature)diff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex b626f1dc3..2690d066a 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -124,6 +124,9 @@ means that care must be taken that the adapter is accessed at `forward` runtime,Another potential fix is to more natively support a "variadic" input field, where the input is a list of dictionaries,or a big dictionary, and have each adatper format it accordingly.+Trajectories also affect meta-programming modules that view the trace later. It's inefficient O(n^2) to view the+trace of every module repeating the prefix.+TOPIC 02: Handling default arguments in the Tool class.@@ -140,4 +143,13 @@ TOPIC 04: Default behavior when the trajectory gets too long.TOPIC 05: Adding more structure around how the instruction is formatted.* Concretely, it's now a string, so an optimizer can and does rewrite it freely.* An alternative would be to add more structure, such that a certain template is fixed but values are variable?+++TOPIC 06: Idiomatically allowing tools that maintain state across iterations, but not across different `forward` calls.+ * So the tool would be newly initialized at the start of each `forward` call, but maintain state across iterations.+ * This is pretty useful for allowing the agent to keep notes or count certain things, etc.++TOPIC 07: Make max_iters a bit more expressive.+ * Allow passing `max_iters` in forward to overwrite the default.+ * Get rid of `last_iteration: bool` in the format function. It's not necessary now."""diff --git a/dspy/predict/retry.py b/dspy/predict/retry.pyindex b515dfba7..66542ba43 100644--- a/dspy/predict/retry.py+++ b/dspy/predict/retry.py@@ -1,74 +1,74 @@-import copy+# import copy-import dspy+# import dspy-from .predict import Predict+# from .predict import Predict-class Retry(Predict):- def __init__(self, module):- super().__init__(module.signature)- self.module = module- self.original_signature = module.extended_signature if isinstance(module, dspy.ChainOfThought) else module.signature- self.original_forward = module.forward- self.new_signature = self._create_new_signature(self.original_signature)+# class Retry(Predict):+# def __init__(self, module):+# super().__init__(module.signature)+# self.module = module+# self.original_signature = module.signature+# self.original_forward = module.forward+# self.new_signature = self._create_new_signature(self.original_signature)- def _create_new_signature(self, signature):- # Add "Past" input fields for each output field- for key, value in signature.output_fields.items():- actual_prefix = value.json_schema_extra["prefix"].split(":")[0] + ":"- signature = signature.append(f"past_{key}", dspy.InputField(- prefix="Previous " + actual_prefix,- desc=f"past {actual_prefix[:-1]} with errors",- format=value.json_schema_extra.get("format"),- ))+# def _create_new_signature(self, signature):+# # Add "Past" input fields for each output field+# for key, value in signature.output_fields.items():+# actual_prefix = value.json_schema_extra["prefix"].split(":")[0] + ":"+# signature = signature.append(f"past_{key}", dspy.InputField(+# prefix="Previous " + actual_prefix,+# desc=f"past {actual_prefix[:-1]} with errors",+# format=value.json_schema_extra.get("format"),+# ))- signature = signature.append("feedback", dspy.InputField(- prefix="Instructions:",- desc="Some instructions you must satisfy",- format=str,- ))+# signature = signature.append("feedback", dspy.InputField(+# prefix="Instructions:",+# desc="Some instructions you must satisfy",+# format=str,+# ))- return signature+# return signature- def forward(self, *, past_outputs, **kwargs):- # Take into account the possible new signature, as in TypedPredictor- new_signature = kwargs.pop("new_signature", None)- if new_signature:- self.original_signature = new_signature- self.new_signature = self._create_new_signature(self.original_signature)+# def forward(self, *, past_outputs, **kwargs):+# # Take into account the possible new signature, as in TypedPredictor+# new_signature = kwargs.pop("new_signature", None)+# if new_signature:+# self.original_signature = new_signature+# self.new_signature = self._create_new_signature(self.original_signature)- # Convert the dict past_outputs={"answer": ...} to kwargs- # {past_answer=..., ...}- for key, value in past_outputs.items():- past_key = f"past_{key}"- if past_key in self.new_signature.input_fields:- kwargs[past_key] = value- # Tell the wrapped module to use the new signature.- # Note: This only works if the wrapped module is a Predict or ChainOfThought.- kwargs["new_signature"] = self.new_signature- return self.original_forward(**kwargs)+# # Convert the dict past_outputs={"answer": ...} to kwargs+# # {past_answer=..., ...}+# for key, value in past_outputs.items():+# past_key = f"past_{key}"+# if past_key in self.new_signature.input_fields:+# kwargs[past_key] = value+# # Tell the wrapped module to use the new signature.+# # Note: This only works if the wrapped module is a Predict or ChainOfThought.+# kwargs["new_signature"] = self.new_signature+# return self.original_forward(**kwargs)- def __call__(self, **kwargs):- copy.deepcopy(kwargs)- kwargs["_trace"] = False- kwargs.setdefault("demos", self.demos if self.demos is not None else [])+# def __call__(self, **kwargs):+# copy.deepcopy(kwargs)+# kwargs["_trace"] = False+# kwargs.setdefault("demos", self.demos if self.demos is not None else [])- # perform backtracking- if dspy.settings.backtrack_to == self:- for key, value in dspy.settings.backtrack_to_args.items():- kwargs.setdefault(key, value)- pred = self.forward(**kwargs)- else:- pred = self.module(**kwargs)+# # perform backtracking+# if dspy.settings.backtrack_to == self:+# for key, value in dspy.settings.backtrack_to_args.items():+# kwargs.setdefault(key, value)+# pred = self.forward(**kwargs)+# else:+# pred = self.module(**kwargs)- # now pop multiple reserved keys- # NOTE(shangyin) past_outputs seems not useful to include in demos,- # therefore dropped- for key in ["_trace", "demos", "signature", "new_signature", "config", "lm", "past_outputs"]:- kwargs.pop(key, None)+# # now pop multiple reserved keys+# # NOTE(shangyin) past_outputs seems not useful to include in demos,+# # therefore dropped+# for key in ["_trace", "demos", "signature", "new_signature", "config", "lm", "past_outputs"]:+# kwargs.pop(key, None)- if dspy.settings.trace is not None:- trace = dspy.settings.trace- trace.append((self, {**kwargs}, pred))- return pred+# if dspy.settings.trace is not None:+# trace = dspy.settings.trace+# trace.append((self, {**kwargs}, pred))+# return preddiff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex a612d02a6..8e456cf53 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -1,344 +1,344 @@-import inspect-import logging-import uuid-from typing import Any--# import dspy.dsp as dsp-import dspy--logger = logging.getLogger(__name__)-#################### Assertion Helpers ####################---def _build_error_msg(feedback_msgs):- """Build an error message from a list of feedback messages."""- return "\n".join([msg for msg in feedback_msgs])---#################### Assertion Exceptions ####################---class DSPyAssertionError(AssertionError):- """Custom exception raised when a DSPy `Assert` fails."""-- def __init__(- self,- id: str,- msg: str,- target_module: Any = None,- state: Any = None,- is_metric: bool = False,- ) -> None:- super().__init__(msg)- self.id = id- self.msg = msg- self.target_module = target_module- self.state = state- self.is_metric = is_metric---class DSPySuggestionError(AssertionError):- """Custom exception raised when a DSPy `Suggest` fails."""-- def __init__(- self,- id: str,- msg: str,- target_module: Any = None,- state: Any = None,- is_metric: bool = False,- ) -> None:- super().__init__(msg)- self.id = id- self.msg = msg- self.target_module = target_module- self.state = state- self.is_metric = is_metric---#################### Assertion Primitives ####################---class Constraint:- def __init__(- self,- result: bool,- msg: str = "",- target_module=None,- is_metric: bool = False,- ):- self.id = str(uuid.uuid4())- self.result = result- self.msg = msg- self.target_module = target_module- self.is_metric = is_metric+# import inspect+# import logging+# import uuid+# from typing import Any++# # import dspy.dsp as dsp+# import dspy++# logger = logging.getLogger(__name__)+# #################### Assertion Helpers ####################+++# def _build_error_msg(feedback_msgs):+# """Build an error message from a list of feedback messages."""+# return "\n".join([msg for msg in feedback_msgs])+++# #################### Assertion Exceptions ####################+++# class DSPyAssertionError(AssertionError):+# """Custom exception raised when a DSPy `Assert` fails."""++# def __init__(+# self,+# id: str,+# msg: str,+# target_module: Any = None,+# state: Any = None,+# is_metric: bool = False,+# ) -> None:+# super().__init__(msg)+# self.id = id+# self.msg = msg+# self.target_module = target_module+# self.state = state+# self.is_metric = is_metric+++# class DSPySuggestionError(AssertionError):+# """Custom exception raised when a DSPy `Suggest` fails."""++# def __init__(+# self,+# id: str,+# msg: str,+# target_module: Any = None,+# state: Any = None,+# is_metric: bool = False,+# ) -> None:+# super().__init__(msg)+# self.id = id+# self.msg = msg+# self.target_module = target_module+# self.state = state+# self.is_metric = is_metric+++# #################### Assertion Primitives ####################+++# class Constraint:+# def __init__(+# self,+# result: bool,+# msg: str = "",+# target_module=None,+# is_metric: bool = False,+# ):+# self.id = str(uuid.uuid4())+# self.result = result+# self.msg = msg+# self.target_module = target_module+# self.is_metric = is_metric- self.__call__()+# self.__call__()-class Assert(Constraint):- """DSPy Assertion"""+# class Assert(Constraint):+# """DSPy Assertion"""- def __call__(self) -> bool:- if isinstance(self.result, bool):- if self.result:- return True- elif dspy.settings.bypass_assert:- logger.error(f"AssertionError: {self.msg}")- return True- else:- logger.error(f"AssertionError: {self.msg}")- raise DSPyAssertionError(- id=self.id,- msg=self.msg,- target_module=self.target_module,- state=dspy.settings.trace,- is_metric=self.is_metric,- )- else:- raise ValueError("Assertion function should always return [bool]")+# def __call__(self) -> bool:+# if isinstance(self.result, bool):+# if self.result:+# return True+# elif dspy.settings.bypass_assert:+# logger.error(f"AssertionError: {self.msg}")+# return True+# else:+# logger.error(f"AssertionError: {self.msg}")+# raise DSPyAssertionError(+# id=self.id,+# msg=self.msg,+# target_module=self.target_module,+# state=dspy.settings.trace,+# is_metric=self.is_metric,+# )+# else:+# raise ValueError("Assertion function should always return [bool]")-class Suggest(Constraint):- """DSPy Suggestion"""+# class Suggest(Constraint):+# """DSPy Suggestion"""- def __call__(self) -> Any:- if isinstance(self.result, bool):- if self.result:- return True- elif dspy.settings.bypass_suggest:- logger.info(f"SuggestionFailed: {self.msg}")- return True- else:- logger.info(f"SuggestionFailed: {self.msg}")- raise DSPySuggestionError(- id=self.id,- msg=self.msg,- target_module=self.target_module,- state=dspy.settings.trace,- is_metric=self.is_metric,- )- else:- raise ValueError("Suggestion function should always return [bool]")---#################### Assertion Handlers ####################---def noop_handler(func):- """Handler to bypass assertions and suggestions.-- Now both assertions and suggestions will become noops.- """-- def wrapper(*args, **kwargs):- with dspy.settings.context(bypass_assert=True, bypass_suggest=True):- return func(*args, **kwargs)-- return wrapper---def bypass_suggest_handler(func):- """Handler to bypass suggest only.-- If a suggestion fails, it will be logged but not raised.- And If an assertion fails, it will be raised.- """-- def wrapper(*args, **kwargs):- with dspy.settings.context(bypass_suggest=True, bypass_assert=False):- return func(*args, **kwargs)-- return wrapper---def bypass_assert_handler(func):- """Handler to bypass assertion only.-- If a assertion fails, it will be logged but not raised.- And If an assertion fails, it will be raised.- """-- def wrapper(*args, **kwargs):- with dspy.settings.context(bypass_assert=True):- return func(*args, **kwargs)-- return wrapper---def assert_no_except_handler(func):- """Handler to ignore assertion failure and return None."""-- def wrapper(*args, **kwargs):- try:- return func(*args, **kwargs)- except DSPyAssertionError:- return None-- return wrapper---def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):- """Handler for backtracking suggestion and assertion.-- Re-run the latest predictor up to `max_backtracks` times,- with updated signature if an assertion fails. updated signature adds a new- input field to the signature, which is the feedback.- """-- def wrapper(*args, **kwargs):- error_msg, result = None, None- with dspy.settings.lock:- dspy.settings.backtrack_to = None- dspy.settings.suggest_failures = 0- dspy.settings.assert_failures = 0-- # Predictor -> List[feedback_msg]- dspy.settings.predictor_feedbacks = {}-- current_error = None- for i in range(max_backtracks + 1):- if i > 0 and dspy.settings.backtrack_to is not None:- # generate values for new fields- feedback_msg = _build_error_msg(- dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to],- )-- dspy.settings.backtrack_to_args = {- "feedback": feedback_msg,- "past_outputs": past_outputs,- }-- # if last backtrack: ignore suggestion errors- if i == max_backtracks:- if isinstance(current_error, DSPyAssertionError):- raise current_error- dspy.settings.trace.clear()- result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else None- break- else:- try:- dspy.settings.trace.clear()- result = func(*args, **kwargs)- break- except (DSPySuggestionError, DSPyAssertionError) as e:- if not current_error:- current_error = e- _error_id, error_msg, error_target_module, error_state = (- e.id,- e.msg,- e.target_module,- e.state[-1],- )-- # increment failure count depending on type of error- if isinstance(e, DSPySuggestionError) and e.is_metric:- dspy.settings.suggest_failures += 1- elif isinstance(e, DSPyAssertionError) and e.is_metric:- dspy.settings.assert_failures += 1-- if dspy.settings.trace:- if error_target_module:- for i in range(len(dspy.settings.trace) - 1, -1, -1):- trace_element = dspy.settings.trace[i]- mod = trace_element[0]- if mod == error_target_module:- error_state = e.state[i]- dspy.settings.backtrack_to = mod- break- else:- dspy.settings.backtrack_to = dspy.settings.trace[-1][0]-- if dspy.settings.backtrack_to is None:- logger.error("Module not found in trace. If passing a DSPy Signature, please specify the intended module for the assertion (e.g., use `target_module = self.my_module(my_signature)` instead of `target_module = my_signature`).")-- # save unique feedback message for predictor- if error_msg not in dspy.settings.predictor_feedbacks.setdefault(- dspy.settings.backtrack_to,- [],- ):- dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to].append(error_msg)-- # use `new_signature` if available (CoT)- if hasattr(error_state[0], 'new_signature'):- output_fields = error_state[0].new_signature.output_fields- else:- output_fields = error_state[0].signature.output_fields- past_outputs = {}- for field_name in output_fields.keys():- past_outputs[field_name] = getattr(- error_state[2],- field_name,- None,- )-- # save latest failure trace for predictor per suggestion- error_state[1]- error_op = error_state[2].__dict__["_store"]- error_op.pop("_assert_feedback", None)- error_op.pop("_assert_traces", None)-- else:- logger.error(- "UNREACHABLE: No trace available, this should not happen. Is this run time?",- )-- return result-- return wrapper---def handle_assert_forward(assertion_handler, **handler_args):- def forward(self, *args, **kwargs):- args_to_vals = inspect.getcallargs(self._forward, *args, **kwargs)-- # if user has specified a bypass_assert flag, set it- if "bypass_assert" in args_to_vals:- dspy.settings.configure(bypass_assert=args_to_vals["bypass_assert"])-- wrapped_forward = assertion_handler(self._forward, **handler_args)- return wrapped_forward(*args, **kwargs)-- return forward---default_assertion_handler = backtrack_handler---def assert_transform_module(- module,- assertion_handler=default_assertion_handler,- **handler_args,-):- """- Transform a module to handle assertions.- """- if not getattr(module, "forward", False):- raise ValueError(- "Module must have a forward method to have assertions handled.",- )- if getattr(module, "_forward", False):- logger.info(- f"Module {module.__class__.__name__} already has a _forward method. Skipping...",- )- pass # TODO warning: might be overwriting a previous _forward method-- module._forward = module.forward- module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(- module,- )-- if all(- map(lambda p: isinstance(p[1], dspy.retry.Retry), module.named_predictors()),- ):- pass # we already applied the Retry mapping outside- elif all(- map(lambda p: not isinstance(p[1], dspy.retry.Retry), module.named_predictors()),- ):- module.map_named_predictors(dspy.retry.Retry)- else:- raise RuntimeError("Module has mixed predictors, can't apply Retry mapping.")-- module._assert_transformed = True-- return module+# def __call__(self) -> Any:+# if isinstance(self.result, bool):+# if self.result:+# return True+# elif dspy.settings.bypass_suggest:+# logger.info(f"SuggestionFailed: {self.msg}")+# return True+# else:+# logger.info(f"SuggestionFailed: {self.msg}")+# raise DSPySuggestionError(+# id=self.id,+# msg=self.msg,+# target_module=self.target_module,+# state=dspy.settings.trace,+# is_metric=self.is_metric,+# )+# else:+# raise ValueError("Suggestion function should always return [bool]")+++# #################### Assertion Handlers ####################+++# def noop_handler(func):+# """Handler to bypass assertions and suggestions.++# Now both assertions and suggestions will become noops.+# """++# def wrapper(*args, **kwargs):+# with dspy.settings.context(bypass_assert=True, bypass_suggest=True):+# return func(*args, **kwargs)++# return wrapper+++# def bypass_suggest_handler(func):+# """Handler to bypass suggest only.++# If a suggestion fails, it will be logged but not raised.+# And If an assertion fails, it will be raised.+# """++# def wrapper(*args, **kwargs):+# with dspy.settings.context(bypass_suggest=True, bypass_assert=False):+# return func(*args, **kwargs)++# return wrapper+++# def bypass_assert_handler(func):+# """Handler to bypass assertion only.++# If a assertion fails, it will be logged but not raised.+# And If an assertion fails, it will be raised.+# """++# def wrapper(*args, **kwargs):+# with dspy.settings.context(bypass_assert=True):+# return func(*args, **kwargs)++# return wrapper+++# def assert_no_except_handler(func):+# """Handler to ignore assertion failure and return None."""++# def wrapper(*args, **kwargs):+# try:+# return func(*args, **kwargs)+# except DSPyAssertionError:+# return None++# return wrapper+++# def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):+# """Handler for backtracking suggestion and assertion.++# Re-run the latest predictor up to `max_backtracks` times,+# with updated signature if an assertion fails. updated signature adds a new+# input field to the signature, which is the feedback.+# """++# def wrapper(*args, **kwargs):+# error_msg, result = None, None+# with dspy.settings.lock:+# dspy.settings.backtrack_to = None+# dspy.settings.suggest_failures = 0+# dspy.settings.assert_failures = 0++# # Predictor -> List[feedback_msg]+# dspy.settings.predictor_feedbacks = {}++# current_error = None+# for i in range(max_backtracks + 1):+# if i > 0 and dspy.settings.backtrack_to is not None:+# # generate values for new fields+# feedback_msg = _build_error_msg(+# dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to],+# )++# dspy.settings.backtrack_to_args = {+# "feedback": feedback_msg,+# "past_outputs": past_outputs,+# }++# # if last backtrack: ignore suggestion errors+# if i == max_backtracks:+# if isinstance(current_error, DSPyAssertionError):+# raise current_error+# dspy.settings.trace.clear()+# result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else None+# break+# else:+# try:+# dspy.settings.trace.clear()+# result = func(*args, **kwargs)+# break+# except (DSPySuggestionError, DSPyAssertionError) as e:+# if not current_error:+# current_error = e+# _error_id, error_msg, error_target_module, error_state = (+# e.id,+# e.msg,+# e.target_module,+# e.state[-1],+# )++# # increment failure count depending on type of error+# if isinstance(e, DSPySuggestionError) and e.is_metric:+# dspy.settings.suggest_failures += 1+# elif isinstance(e, DSPyAssertionError) and e.is_metric:+# dspy.settings.assert_failures += 1++# if dspy.settings.trace:+# if error_target_module:+# for i in range(len(dspy.settings.trace) - 1, -1, -1):+# trace_element = dspy.settings.trace[i]+# mod = trace_element[0]+# if mod == error_target_module:+# error_state = e.state[i]+# dspy.settings.backtrack_to = mod+# break+# else:+# dspy.settings.backtrack_to = dspy.settings.trace[-1][0]++# if dspy.settings.backtrack_to is None:+# logger.error("Module not found in trace. If passing a DSPy Signature, please specify the intended module for the assertion (e.g., use `target_module = self.my_module(my_signature)` instead of `target_module = my_signature`).")++# # save unique feedback message for predictor+# if error_msg not in dspy.settings.predictor_feedbacks.setdefault(+# dspy.settings.backtrack_to,+# [],+# ):+# dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to].append(error_msg)++# # use `new_signature` if available (CoT)+# if hasattr(error_state[0], 'new_signature'):+# output_fields = error_state[0].new_signature.output_fields+# else:+# output_fields = error_state[0].signature.output_fields+# past_outputs = {}+# for field_name in output_fields.keys():+# past_outputs[field_name] = getattr(+# error_state[2],+# field_name,+# None,+# )++# # save latest failure trace for predictor per suggestion+# error_state[1]+# error_op = error_state[2].__dict__["_store"]+# error_op.pop("_assert_feedback", None)+# error_op.pop("_assert_traces", None)++# else:+# logger.error(+# "UNREACHABLE: No trace available, this should not happen. Is this run time?",+# )++# return result++# return wrapper+++# def handle_assert_forward(assertion_handler, **handler_args):+# def forward(self, *args, **kwargs):+# args_to_vals = inspect.getcallargs(self._forward, *args, **kwargs)++# # if user has specified a bypass_assert flag, set it+# if "bypass_assert" in args_to_vals:+# dspy.settings.configure(bypass_assert=args_to_vals["bypass_assert"])++# wrapped_forward = assertion_handler(self._forward, **handler_args)+# return wrapped_forward(*args, **kwargs)++# return forward+++# default_assertion_handler = backtrack_handler+++# def assert_transform_module(+# module,+# assertion_handler=default_assertion_handler,+# **handler_args,+# ):+# """+# Transform a module to handle assertions.+# """+# if not getattr(module, "forward", False):+# raise ValueError(+# "Module must have a forward method to have assertions handled.",+# )+# if getattr(module, "_forward", False):+# logger.info(+# f"Module {module.__class__.__name__} already has a _forward method. Skipping...",+# )+# pass # TODO warning: might be overwriting a previous _forward method++# module._forward = module.forward+# module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(+# module,+# )++# if all(+# map(lambda p: isinstance(p[1], dspy.retry.Retry), module.named_predictors()),+# ):+# pass # we already applied the Retry mapping outside+# elif all(+# map(lambda p: not isinstance(p[1], dspy.retry.Retry), module.named_predictors()),+# ):+# module.map_named_predictors(dspy.retry.Retry)+# else:+# raise RuntimeError("Module has mixed predictors, can't apply Retry mapping.")++# module._assert_transformed = True++# return modulediff --git a/dspy/primitives/module.py b/dspy/primitives/module.pyindex c2146301c..8bec91620 100644--- a/dspy/primitives/module.py+++ b/dspy/primitives/module.py@@ -256,14 +256,15 @@ class BaseModule:def postprocess_parameter_name(name, value):- # For ChainOfThought backward compatibility, remove ending ._predict if it's there- if name.endswith("._predict"):- name = name[:-9]+ return name+ # # For ChainOfThought backward compatibility, remove ending ._predict if it's there+ # if name.endswith("._predict"):+ # name = name[:-9]- if name.endswith(".self"):- name = name[:-5]+ # if name.endswith(".self"):+ # name = name[:-5]- if name == "_predict":- return "self"+ # if name == "_predict":+ # return "self"- return name+ # return namediff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex a8b2d1346..a8b9676da 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -1,10 +1,10 @@from dspy.utils.callback import with_callbacksimport magicattr-import dspyfrom dspy.predict.parallel import Parallel-from dspy.primitives.assertions import *from dspy.primitives.module import BaseModule+# import dspy+# from dspy.primitives.assertions import *class ProgramMeta(type):@@ -32,28 +32,16 @@ class Module(BaseModule, metaclass=ProgramMeta):return [param for _, param in self.named_predictors()]def set_lm(self, lm):- if not dspy.settings.experimental:- raise ValueError(- "Setting or getting the LM of a program is an experimental feature. Please enable the "- "'dspy.settings.experimental' flag to use these features."- )-for _, param in self.named_predictors():param.lm = lmdef get_lm(self):- if not dspy.settings.experimental:- raise ValueError(- "Setting or getting the LM of a program is an experimental feature. Please enable the "- "'dspy.settings.experimental' flag to use these features."- )-all_used_lms = [param.lm for _, param in self.named_predictors()]if len(set(all_used_lms)) == 1:return all_used_lms[0]- raise ValueError("Multiple LMs are being used in the module.")+ raise ValueError("Multiple LMs are being used in the module. There's no unique LM to return.")def __repr__(self):s = []@@ -69,13 +57,13 @@ class Module(BaseModule, metaclass=ProgramMeta):set_attribute_by_name(self, name, func(predictor))return self- def activate_assertions(self, handler=backtrack_handler, **handler_args):- """- Activates assertions for the module.- The default handler is the backtrack_handler.- """- assert_transform_module(self, handler, **handler_args)- return self+ # def activate_assertions(self, handler=backtrack_handler, **handler_args):+ # """+ # Activates assertions for the module.+ # The default handler is the backtrack_handler.+ # """+ # assert_transform_module(self, handler, **handler_args)+ # return self# def __deepcopy__(self, memo):# # memo is a dict of id's to copies already made during the current call@@ -103,7 +91,7 @@ class Module(BaseModule, metaclass=ProgramMeta):return_failed_examples: bool = False,provide_traceback: bool = False,disable_progress_bar: bool = False,- ) -> Any:+ ):"""Processes a list of dspy.Example instances in parallel using the Parallel module.diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.pyindex de8cc9f0e..feeecf9e3 100644--- a/dspy/teleprompt/copro_optimizer.py+++ b/dspy/teleprompt/copro_optimizer.py@@ -113,16 +113,12 @@ class COPRO(Teleprompter):logger.debug(f"p: {list(signature.fields.values())[-1].json_schema_extra['prefix']}")def _get_signature(self, predictor):- if hasattr(predictor, "extended_signature"):- return predictor.extended_signature- elif hasattr(predictor, "signature"):- return predictor.signature+ assert hasattr(predictor, "signature")+ return predictor.signaturedef _set_signature(self, predictor, updated_signature):- if hasattr(predictor, "extended_signature"):- predictor.extended_signature = updated_signature- elif hasattr(predictor, "signature"):- predictor.signature = updated_signature+ assert hasattr(predictor, "signature")+ predictor.signature = updated_signaturedef compile(self, student, *, trainset, eval_kwargs):"""diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pyindex d763d74da..bdc6a529a 100644--- a/dspy/teleprompt/utils.py+++ b/dspy/teleprompt/utils.py@@ -257,18 +257,13 @@ def get_prompt_model(prompt_model):return dspy.settings.lmdef get_signature(predictor):- if hasattr(predictor, "extended_signature"):- return predictor.extended_signature- elif hasattr(predictor, "signature"):- return predictor.signature- return None+ assert hasattr(predictor, "signature")+ return predictor.signaturedef set_signature(predictor, updated_signature):- if hasattr(predictor, "extended_signature"):- predictor.extended_signature = updated_signature- elif hasattr(predictor, "signature"):- predictor.signature = updated_signature+ assert hasattr(predictor, "signature")+ predictor.signature = updated_signaturedef create_n_fewshot_demo_sets(diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pyindex 04f73bb51..c050819ef 100644--- a/tests/predict/test_chain_of_thought.py+++ b/tests/predict/test_chain_of_thought.py@@ -7,7 +7,7 @@ def test_initialization_with_string_signature():lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])dspy.settings.configure(lm=lm)predict = ChainOfThought("question -> answer")- assert list(predict.extended_signature.output_fields.keys()) == [+ assert list(predict.predict.signature.output_fields.keys()) == ["reasoning","answer",]diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.pyindex ba28e9043..eb1c48b06 100644--- a/tests/predict/test_retry.py+++ b/tests/predict/test_retry.py@@ -1,86 +1,48 @@-import functools+# import functools-import pydantic+# import pydantic-import dspy-from dspy.primitives.assertions import assert_transform_module, backtrack_handler-from dspy.utils import DummyLM+# import dspy+# from dspy.primitives.assertions import assert_transform_module, backtrack_handler+# from dspy.utils import DummyLM-def test_retry_simple():- predict = dspy.Predict("question -> answer")- retry_module = dspy.Retry(predict)+# def test_retry_simple():+# predict = dspy.Predict("question -> answer")+# retry_module = dspy.Retry(predict)- # Test Retry has created the correct new signature- for field in predict.signature.output_fields:- assert f"past_{field}" in retry_module.new_signature.input_fields- assert "feedback" in retry_module.new_signature.input_fields+# # Test Retry has created the correct new signature+# for field in predict.signature.output_fields:+# assert f"past_{field}" in retry_module.new_signature.input_fields+# assert "feedback" in retry_module.new_signature.input_fields- lm = DummyLM([{"answer": "blue"}])- dspy.settings.configure(lm=lm)- result = retry_module.forward(- question="What color is the sky?",- past_outputs={"answer": "red"},- feedback="Try harder",- )- assert result.answer == "blue"---def test_retry_forward_with_feedback():- # First we make a mistake, then we fix it- lm = DummyLM([{"answer": "red"}, {"answer": "blue"}])- dspy.settings.configure(lm=lm, trace=[])-- class SimpleModule(dspy.Module):- def __init__(self):- super().__init__()- self.predictor = dspy.Predict("question -> answer")-- def forward(self, **kwargs):- result = self.predictor(**kwargs)- print(f"SimpleModule got {result.answer=}")- dspy.Suggest(result.answer == "blue", "Please think harder")- return result-- program = SimpleModule()- program = assert_transform_module(- program.map_named_predictors(dspy.Retry),- functools.partial(backtrack_handler, max_backtracks=1),- )-- result = program(question="What color is the sky?")-- assert result.answer == "blue"+# lm = DummyLM([{"answer": "blue"}])+# dspy.settings.configure(lm=lm)+# result = retry_module.forward(+# question="What color is the sky?",+# past_outputs={"answer": "red"},+# feedback="Try harder",+# )+# assert result.answer == "blue"-# def test_retry_forward_with_typed_predictor():+# def test_retry_forward_with_feedback():# # First we make a mistake, then we fix it-# lm = DummyLM([{"output": '{"answer":"red"}'}, {"output": '{"answer":"blue"}'}])+# lm = DummyLM([{"answer": "red"}, {"answer": "blue"}])# dspy.settings.configure(lm=lm, trace=[])-# class AnswerQuestion(dspy.Signature):-# """Answer questions with succinct responses."""--# class Input(pydantic.BaseModel):-# question: str--# class Output(pydantic.BaseModel):-# answer: str--# input: Input = dspy.InputField()-# output: Output = dspy.OutputField()--# class QuestionAnswerer(dspy.Module):+# class SimpleModule(dspy.Module):# def __init__(self):# super().__init__()-# self.answer_question = dspy.TypedPredictor(AnswerQuestion)+# self.predictor = dspy.Predict("question -> answer")# def forward(self, **kwargs):-# result = self.answer_question(input=AnswerQuestion.Input(**kwargs)).output+# result = self.predictor(**kwargs)+# print(f"SimpleModule got {result.answer=}")# dspy.Suggest(result.answer == "blue", "Please think harder")# return result-# program = QuestionAnswerer()+# program = SimpleModule()# program = assert_transform_module(# program.map_named_predictors(dspy.Retry),# functools.partial(backtrack_handler, max_backtracks=1),@@ -89,3 +51,41 @@ def test_retry_forward_with_feedback():# result = program(question="What color is the sky?")# assert result.answer == "blue"+++# # def test_retry_forward_with_typed_predictor():+# # # First we make a mistake, then we fix it+# # lm = DummyLM([{"output": '{"answer":"red"}'}, {"output": '{"answer":"blue"}'}])+# # dspy.settings.configure(lm=lm, trace=[])++# # class AnswerQuestion(dspy.Signature):+# # """Answer questions with succinct responses."""++# # class Input(pydantic.BaseModel):+# # question: str++# # class Output(pydantic.BaseModel):+# # answer: str++# # input: Input = dspy.InputField()+# # output: Output = dspy.OutputField()++# # class QuestionAnswerer(dspy.Module):+# # def __init__(self):+# # super().__init__()+# # self.answer_question = dspy.TypedPredictor(AnswerQuestion)++# # def forward(self, **kwargs):+# # result = self.answer_question(input=AnswerQuestion.Input(**kwargs)).output+# # dspy.Suggest(result.answer == "blue", "Please think harder")+# # return result++# # program = QuestionAnswerer()+# # program = assert_transform_module(+# # program.map_named_predictors(dspy.Retry),+# # functools.partial(backtrack_handler, max_backtracks=1),+# # )++# # result = program(question="What color is the sky?")++# # assert result.answer == "blue"diff --git a/tests/primitives/test_module.py b/tests/primitives/test_module.pyindex 5e6790d8e..d0532fca1 100644--- a/tests/primitives/test_module.py+++ b/tests/primitives/test_module.py@@ -51,8 +51,8 @@ def test_deepcopy_with_nested_modules():def test_save_and_load_with_json(tmp_path):model = dspy.ChainOfThought(dspy.Signature("q -> a"))- model._predict.signature = model._predict.signature.with_instructions("You are a helpful assistant.")- model._predict.demos = [+ model.predict.signature = model.predict.signature.with_instructions("You are a helpful assistant.")+ model.predict.demos = [dspy.Example(q="What is the capital of France?", a="Paris", reasoning="n/a").with_inputs("q", "a")]save_path = tmp_path / "model.json"@@ -60,8 +60,8 @@ def test_save_and_load_with_json(tmp_path):new_model = dspy.ChainOfThought(dspy.Signature("q -> a"))new_model.load(save_path)- assert str(new_model.signature) == str(model.signature)- assert new_model.demos[0] == model.demos[0].toDict()+ assert str(new_model.predict.signature) == str(model.predict.signature)+ assert new_model.predict.demos[0] == model.predict.demos[0].toDict()def test_save_and_load_with_pkl(tmp_path):@@ -96,7 +96,7 @@ def test_save_and_load_with_pkl(tmp_path):optimizer = dspy.BootstrapFewShot(max_bootstrapped_demos=4, max_labeled_demos=4, max_rounds=5, metric=dummy_metric)compiled_cot = optimizer.compile(cot, trainset=trainset)- compiled_cot._predict.signature = compiled_cot._predict.signature.with_instructions("You are a helpful assistant.")+ compiled_cot.predict.signature = compiled_cot.predict.signature.with_instructions("You are a helpful assistant.")save_path = tmp_path / "program.pkl"compiled_cot.save(save_path)@@ -104,5 +104,5 @@ def test_save_and_load_with_pkl(tmp_path):new_cot = dspy.ChainOfThought(MySignature)new_cot.load(save_path)- assert str(new_cot.signature) == str(compiled_cot.signature)- assert new_cot.demos == compiled_cot.demos+ assert str(new_cot.predict.signature) == str(compiled_cot.predict.signature)+ assert new_cot.predict.demos == compiled_cot.predict.demos