Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

updated assertions with thread-safe measure and notebook with proper caching

Arnav Singhviaccd13818f81
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py
index f4d207b2f..60575720b 100644
--- a/dsp/utils/settings.py
+++ b/dsp/utils/settings.py
@@ -15,6 +15,7 @@ class Settings(object):
if cls._instance is None:
cls._instance = super().__new__(cls)
+ cls._instance.lock = threading.Lock()
cls._instance.main_tid = threading.get_ident()
cls._instance.main_stack = []
cls._instance.stack_by_thread = {}
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index 438e869bd..4962c2733 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -179,85 +179,86 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
def wrapper(*args, **kwargs):
error_msg, result = None, None
- dspy.settings.backtrack_to = None
+ with dspy.settings.lock:
+ dspy.settings.backtrack_to = None
- # Predictor -> List[feedback_msg]
- dspy.settings.predictor_feedbacks = {}
+ # Predictor -> List[feedback_msg]
+ dspy.settings.predictor_feedbacks = {}
- for i in range(max_backtracks + 1):
- if i > 0 and dspy.settings.backtrack_to is not None:
- # generate values for new fields
- feedback_msg = _build_error_msg(
- dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to]
- )
-
- dspy.settings.backtrack_to_args = {
- "feedback": feedback_msg,
- "past_outputs": past_outputs,
- }
+ for i in range(max_backtracks + 1):
+ if i > 0 and dspy.settings.backtrack_to is not None:
+ # generate values for new fields
+ feedback_msg = _build_error_msg(
+ dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to]
+ )
- # if last backtrack: ignore suggestion errors
- if i == max_backtracks:
- result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else None
- break
+ dspy.settings.backtrack_to_args = {
+ "feedback": feedback_msg,
+ "past_outputs": past_outputs,
+ }
- else:
- try:
- result = func(*args, **kwargs)
+ # if last backtrack: ignore suggestion errors
+ if i == max_backtracks:
+ result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else None
break
- except DSPySuggestionError as e:
- suggest_id, error_msg, suggest_target_module, error_state = (
- e.id,
- e.msg,
- e.target_module,
- e.state[-1],
- )
-
- if dsp.settings.trace:
- if suggest_target_module:
- for i in range(len(dsp.settings.trace) - 1, -1, -1):
- trace_element = dsp.settings.trace[i]
- mod = trace_element[0]
- if mod.signature == suggest_target_module:
- dspy.settings.backtrack_to = mod
- break
- else:
- dspy.settings.backtrack_to = dsp.settings.trace[-1][0]
- if dspy.settings.backtrack_to is None:
- logger.error("Specified module not found in trace")
+ else:
+ try:
+ result = func(*args, **kwargs)
+ break
+ except DSPySuggestionError as e:
+ suggest_id, error_msg, suggest_target_module, error_state = (
+ e.id,
+ e.msg,
+ e.target_module,
+ e.state[-1],
+ )
- # save unique feedback message for predictor
- if (
- error_msg
- not in dspy.settings.predictor_feedbacks.setdefault(
- dspy.settings.backtrack_to, []
- )
- ):
- dspy.settings.predictor_feedbacks[
- dspy.settings.backtrack_to
- ].append(error_msg)
-
- output_fields = vars(error_state[0].signature.signature)
- past_outputs = {}
- for field_name, field_obj in output_fields.items():
- if isinstance(field_obj, dspy.OutputField):
- past_outputs[field_name] = getattr(
- error_state[2], field_name, None
+ if dsp.settings.trace:
+ if suggest_target_module:
+ for i in range(len(dsp.settings.trace) - 1, -1, -1):
+ trace_element = dsp.settings.trace[i]
+ mod = trace_element[0]
+ if mod.signature == suggest_target_module:
+ dspy.settings.backtrack_to = mod
+ break
+ else:
+ dspy.settings.backtrack_to = dsp.settings.trace[-1][0]
+
+ if dspy.settings.backtrack_to is None:
+ logger.error("Specified module not found in trace")
+
+ # save unique feedback message for predictor
+ if (
+ error_msg
+ not in dspy.settings.predictor_feedbacks.setdefault(
+ dspy.settings.backtrack_to, []
)
+ ):
+ dspy.settings.predictor_feedbacks[
+ dspy.settings.backtrack_to
+ ].append(error_msg)
+
+ output_fields = vars(error_state[0].signature.signature)
+ past_outputs = {}
+ for field_name, field_obj in output_fields.items():
+ if isinstance(field_obj, dspy.OutputField):
+ past_outputs[field_name] = getattr(
+ error_state[2], field_name, None
+ )
+
+ # save latest failure trace for predictor per suggestion
+ error_ip = error_state[1]
+ error_op = error_state[2].__dict__["_store"]
+ error_op.pop("_assert_feedback", None)
+ error_op.pop("_assert_traces", None)
- # save latest failure trace for predictor per suggestion
- error_ip = error_state[1]
- error_op = error_state[2].__dict__["_store"]
- error_op.pop("_assert_feedback", None)
- error_op.pop("_assert_traces", None)
-
- else:
- logger.error(
- f"UNREACHABLE: No trace available, this should not happen. Is this run time?"
- )
+ else:
+ logger.error(
+ f"UNREACHABLE: No trace available, this should not happen. Is this run time?"
+ )
- return result
+ return result
return wrapper
diff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynb
index c6a216af9..236837b0c 100644
--- a/examples/longformqa/longformqa_assertions.ipynb
+++ b/examples/longformqa/longformqa_assertions.ipynb
@@ -29,9 +29,31 @@
"metadata": {},
"source": [
"### 0] Setting Up\n",
+ "Let's begin by setting things up. The snippet below will retrieve the cached requests for the task."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!git clone https://huggingface.co/arnavs11/DSPy_LongFormQA_Cache\n",
+ "%cd DSPy_LongFormQA_Cache/\n",
+ "!git checkout master\n",
+ "%cd ..\n",
+ "import os\n",
+ "repo_clone_path = '/content/DSPy_LongFormQA_Cache'\n",
"\n",
- "\n",
- "Let's begin by setting things up. The snippet below will also install **DSPy** if it's not there already."
+ "# Set up the cache for this notebook\n",
+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will also install **DSPy** if it's not there already."
]
},
{
@@ -57,14 +79,11 @@
"if repo_path not in sys.path:\n",
" sys.path.append(repo_path)\n",
"\n",
- "# Set up the cache for this notebook\n",
- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = os.path.join(repo_path, 'cache')\n",
- "\n",
"import pkg_resources # Install the package if it's not installed\n",
"if not \"dspy-ai\" in {pkg.key for pkg in pkg_resources.working_set}:\n",
" !pip install -U pip\n",
" !pip install dspy-ai\n",
- " # !pip install -e $repo_path\n",
+ " !pip install -e $repo_path\n",
"\n",
"import dspy\n",
"from dspy.predict import Retry\n",