DSPy: The framework for programming—not prompting—language models
Python
updated assertions with thread-safe measure and notebook with proper caching
Arnav Singhviaccd13818f81
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex f4d207b2f..60575720b 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -15,6 +15,7 @@ class Settings(object):if cls._instance is None:cls._instance = super().__new__(cls)+ cls._instance.lock = threading.Lock()cls._instance.main_tid = threading.get_ident()cls._instance.main_stack = []cls._instance.stack_by_thread = {}diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 438e869bd..4962c2733 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -179,85 +179,86 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):def wrapper(*args, **kwargs):error_msg, result = None, None- dspy.settings.backtrack_to = None+ with dspy.settings.lock:+ dspy.settings.backtrack_to = None- # Predictor -> List[feedback_msg]- dspy.settings.predictor_feedbacks = {}+ # Predictor -> List[feedback_msg]+ dspy.settings.predictor_feedbacks = {}- for i in range(max_backtracks + 1):- if i > 0 and dspy.settings.backtrack_to is not None:- # generate values for new fields- feedback_msg = _build_error_msg(- dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to]- )-- dspy.settings.backtrack_to_args = {- "feedback": feedback_msg,- "past_outputs": past_outputs,- }+ for i in range(max_backtracks + 1):+ if i > 0 and dspy.settings.backtrack_to is not None:+ # generate values for new fields+ feedback_msg = _build_error_msg(+ dspy.settings.predictor_feedbacks[dspy.settings.backtrack_to]+ )- # if last backtrack: ignore suggestion errors- if i == max_backtracks:- result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else None- break+ dspy.settings.backtrack_to_args = {+ "feedback": feedback_msg,+ "past_outputs": past_outputs,+ }- else:- try:- result = func(*args, **kwargs)+ # if last backtrack: ignore suggestion errors+ if i == max_backtracks:+ result = bypass_suggest_handler(func)(*args, **kwargs) if bypass_suggest else Nonebreak- except DSPySuggestionError as e:- suggest_id, error_msg, suggest_target_module, error_state = (- e.id,- e.msg,- e.target_module,- e.state[-1],- )-- if dsp.settings.trace:- if suggest_target_module:- for i in range(len(dsp.settings.trace) - 1, -1, -1):- trace_element = dsp.settings.trace[i]- mod = trace_element[0]- if mod.signature == suggest_target_module:- dspy.settings.backtrack_to = mod- break- else:- dspy.settings.backtrack_to = dsp.settings.trace[-1][0]- if dspy.settings.backtrack_to is None:- logger.error("Specified module not found in trace")+ else:+ try:+ result = func(*args, **kwargs)+ break+ except DSPySuggestionError as e:+ suggest_id, error_msg, suggest_target_module, error_state = (+ e.id,+ e.msg,+ e.target_module,+ e.state[-1],+ )- # save unique feedback message for predictor- if (- error_msg- not in dspy.settings.predictor_feedbacks.setdefault(- dspy.settings.backtrack_to, []- )- ):- dspy.settings.predictor_feedbacks[- dspy.settings.backtrack_to- ].append(error_msg)-- output_fields = vars(error_state[0].signature.signature)- past_outputs = {}- for field_name, field_obj in output_fields.items():- if isinstance(field_obj, dspy.OutputField):- past_outputs[field_name] = getattr(- error_state[2], field_name, None+ if dsp.settings.trace:+ if suggest_target_module:+ for i in range(len(dsp.settings.trace) - 1, -1, -1):+ trace_element = dsp.settings.trace[i]+ mod = trace_element[0]+ if mod.signature == suggest_target_module:+ dspy.settings.backtrack_to = mod+ break+ else:+ dspy.settings.backtrack_to = dsp.settings.trace[-1][0]++ if dspy.settings.backtrack_to is None:+ logger.error("Specified module not found in trace")++ # save unique feedback message for predictor+ if (+ error_msg+ not in dspy.settings.predictor_feedbacks.setdefault(+ dspy.settings.backtrack_to, [])+ ):+ dspy.settings.predictor_feedbacks[+ dspy.settings.backtrack_to+ ].append(error_msg)++ output_fields = vars(error_state[0].signature.signature)+ past_outputs = {}+ for field_name, field_obj in output_fields.items():+ if isinstance(field_obj, dspy.OutputField):+ past_outputs[field_name] = getattr(+ error_state[2], field_name, None+ )++ # save latest failure trace for predictor per suggestion+ error_ip = error_state[1]+ error_op = error_state[2].__dict__["_store"]+ error_op.pop("_assert_feedback", None)+ error_op.pop("_assert_traces", None)- # save latest failure trace for predictor per suggestion- error_ip = error_state[1]- error_op = error_state[2].__dict__["_store"]- error_op.pop("_assert_feedback", None)- error_op.pop("_assert_traces", None)-- else:- logger.error(- f"UNREACHABLE: No trace available, this should not happen. Is this run time?"- )+ else:+ logger.error(+ f"UNREACHABLE: No trace available, this should not happen. Is this run time?"+ )- return result+ return resultreturn wrapperdiff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynbindex c6a216af9..236837b0c 100644--- a/examples/longformqa/longformqa_assertions.ipynb+++ b/examples/longformqa/longformqa_assertions.ipynb@@ -29,9 +29,31 @@"metadata": {},"source": ["### 0] Setting Up\n",+ "Let's begin by setting things up. The snippet below will retrieve the cached requests for the task."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "!git clone https://huggingface.co/arnavs11/DSPy_LongFormQA_Cache\n",+ "%cd DSPy_LongFormQA_Cache/\n",+ "!git checkout master\n",+ "%cd ..\n",+ "import os\n",+ "repo_clone_path = '/content/DSPy_LongFormQA_Cache'\n","\n",- "\n",- "Let's begin by setting things up. The snippet below will also install **DSPy** if it's not there already."+ "# Set up the cache for this notebook\n",+ "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We will also install **DSPy** if it's not there already."]},{@@ -57,14 +79,11 @@"if repo_path not in sys.path:\n"," sys.path.append(repo_path)\n","\n",- "# Set up the cache for this notebook\n",- "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = os.path.join(repo_path, 'cache')\n",- "\n","import pkg_resources # Install the package if it's not installed\n","if not \"dspy-ai\" in {pkg.key for pkg in pkg_resources.working_set}:\n"," !pip install -U pip\n"," !pip install dspy-ai\n",- " # !pip install -e $repo_path\n",+ " !pip install -e $repo_path\n","\n","import dspy\n","from dspy.predict import Retry\n",