Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

clean up assertions

manishshettymeea3b2c0768f
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index e945603dc..c4b65ffc2 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -1,13 +1,11 @@
-# Note(shangyin): What abstractions we want to make about assertions?
-# things to consider: what we want to assert, and what would be the syntax?
-# One possible starting point would be constraints on the output of a module
-from typing import Any
+from typing import Any, Callable
import dsp
import dspy
class DSPyAssertionError(AssertionError):
- """DSPy custom error message."""
+ """Custom exception raised when a DSPy `Assert` fails.
+ """
def __init__(self, msg: str, state: Any = None) -> None:
super().__init__(msg)
@@ -16,9 +14,9 @@ class DSPyAssertionError(AssertionError):
class Assert:
- """Compile time assertion."""
+ """DSPy Assertion"""
- def __init__(self, assert_fun, *args, **kwargs):
+ def __init__(self, assert_fun: Callable, *args, **kwargs):
self.assert_fun = assert_fun
self.args = args
@@ -31,7 +29,6 @@ class Assert:
self.kwargs = kwargs
self.__call__()
- # assert fun should always return bool
def __call__(self) -> bool:
result = self.assert_fun(*self.args, **self.kwargs)
if isinstance(result, bool):
@@ -43,118 +40,12 @@ class Assert:
raise ValueError("Assertion function should always return [bool]")
-# we could possibly to have runtime assertions as well
-# class RuntimeAssert(Assert): ...
-
-
-############################# ASSERTION AND BACKTRACKING POLICIES #############################
-
-
-def assert_transform(max_backtracks=2):
- """Decorator that simply re-runs the function if assertion fails,
- up to `max_backtracks` times."""
-
- def wrapper(func):
- def inner(*args, **kwargs):
- for i in range(max_backtracks + 1):
- try:
- return func(*args, **kwargs)
- except DSPyAssertionError as e:
- print(f"{e.msg}")
- if dsp.settings.trace:
- pass
- else:
- print(
- "UNREACHABLE: No trace available, this should not happen. Is this run time?"
- )
-
- return inner
-
- return wrapper
-
-
-def assert_update_transform(max_backtracks=2):
- """Decorator that simply re-runs the function with updated temperature
- if assertion fails, up to `max_backtracks` times."""
-
- def wrapper(func):
- def inner(*args, **kwargs):
- for i in range(max_backtracks + 1):
- lm = dsp.settings.lm
-
- if i > 0 and backtrack_to is not None:
- # print(f"updating temperature to {0.71 + 0.002 * i}")
- lm = lm.copy(temperature=0.71 + 0.002 * i)
-
- new_settings = dict(lm=lm) if i > 0 else {}
-
- try:
- with dsp.settings.context(**new_settings):
- return func(*args, **kwargs)
- except DSPyAssertionError as e:
- print(f"{e.msg}")
-
- if dsp.settings.trace:
- backtrack_to = id(dsp.settings.trace[-1][0])
- else:
- print(
- "UNREACHABLE: No trace available, this should not happen. Is this run time?"
- )
-
- return inner
-
- return wrapper
-
-
-def assert_latest_transform(max_backtracks=2):
- """Decorator that simply re-runs the function but updates the temperature
- only for the latest predictor in the trace if assertion fails, up to `max_backtracks` times.
-
- # NOTE (@manish): the previous assert_update_transform applies temperature change to all predictors in the pipeline.
- # Here, we udpate/pass the new temperature to the *backtrack_to* predictor *only* --> cause a cache miss --> re-run
- # For other predictors, settings remain the same --> cache hit --> no re-run
- """
-
- def wrapper(func):
- def inner(*args, **kwargs):
- backtrack_to = None
-
- for i in range(max_backtracks + 1):
- if i > 0 and backtrack_to is not None:
- print(f"rewinding to {id(backtrack_to)}")
-
- # udpate temperature via "config" attribute of the `backtrack_to` predictor
- predictor_id = id(backtrack_to)
- print(
- f"prev temp @ {predictor_id}: {backtrack_to.get_config().get('temperature', 'default')}"
- )
- backtrack_to.update_config(temperature=0.7 + 0.001 * i)
- print(
- f"new temp @ {predictor_id}: {backtrack_to.get_config()['temperature']}"
- )
- try:
- return func(*args, **kwargs)
- except DSPyAssertionError as e:
- print(f"{e.msg}")
-
- if dsp.settings.trace:
- backtrack_to = dsp.settings.trace[-1][0]
- else:
- print(
- "UNREACHABLE: No trace available, this should not happen. Is this run time?"
- )
-
- return inner
-
- return wrapper
-
-
def assert_latest_feedback_transform(max_backtracks=2):
- """ "Decorator that simply re-runs the function but updates the signature of
- the latest predictor in the trace if assertion fails, up to `max_backtracks` times.
-
- The updated signature passes a new input to the predictor, which is the feedback
- from the failed assertion. This feedback will be used by the predictor to self-repair.
+ """Decorator that defines the backtracking policy for assertions.
+
+ current policy: re-run the latest predictor up to `max_backtracks` times,
+ with updated signature if assertion fails. updated signature adds a new
+ input field to the signature, which is the feedback.
"""
def wrapper(func):
@@ -211,8 +102,6 @@ def assert_latest_feedback_transform(max_backtracks=2):
for i in range(max_backtracks + 1):
if i > 0 and backtrack_to is not None:
- print(f"rewinding to {id(backtrack_to)}")
-
# create a new feedback field
feedback = dspy.InputField(
prefix="Instruction:", desc="Some instructions you must satisfy"
@@ -251,7 +140,6 @@ def assert_latest_feedback_transform(max_backtracks=2):
predictor,
original_forward,
) in extended_predictors_to_original_forward.items():
- print(f"reverting predictor {id(predictor)}")
_revert_predictor_signature(predictor, "feedback")
predictor.forward = original_forward