DSPy: The framework for programming—not prompting—language models
Python
Add timeout-based straggler resubmission in ParallelExecutor (#7914)
Omar Khattaba2fc6a118766
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.pyindex 08dae8459..51d779f83 100644--- a/dspy/utils/parallelizer.py+++ b/dspy/utils/parallelizer.py@@ -4,9 +4,10 @@ import signalimport loggingimport threadingimport traceback+import timeimport contextlib-from tqdm.contrib.logging import logging_redirect_tqdm-from concurrent.futures import ThreadPoolExecutor, as_completed++from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETEDlogger = logging.getLogger(__name__)@@ -19,189 +20,195 @@ class ParallelExecutor:disable_progress_bar=False,provide_traceback=False,compare_results=False,+ timeout=120,+ straggler_limit=3,):- """Offers isolation between the tasks (dspy.settings) irrespective of whether num_threads == 1 or > 1."""+ """+ Offers isolation between the tasks (dspy.settings) irrespective of whether num_threads == 1 or > 1.+ Handles also straggler timeouts.+ """+self.num_threads = num_threads- self.disable_progress_bar = disable_progress_barself.max_errors = max_errors+ self.disable_progress_bar = disable_progress_barself.provide_traceback = provide_tracebackself.compare_results = compare_results+ self.timeout = timeout+ self.straggler_limit = straggler_limitself.error_count = 0self.error_lock = threading.Lock()self.cancel_jobs = threading.Event()def execute(self, function, data):- wrapped_function = self._wrap_function(function)- if self.num_threads == 1:- return self._execute_isolated_single_thread(wrapped_function, data)- else:- return self._execute_multi_thread(wrapped_function, data)+ wrapped = self._wrap_function(function)+ return self._execute_parallel(wrapped, data)- def _wrap_function(self, function):- # Wrap the function with error handling- def wrapped(item):+ def _wrap_function(self, user_function):+ def safe_func(item):if self.cancel_jobs.is_set():return Nonetry:- return function(item)+ return user_function(item)except Exception as e:with self.error_lock:self.error_count += 1- current_error_count = self.error_count- if current_error_count >= self.max_errors:- self.cancel_jobs.set()- raise e+ if self.error_count >= self.max_errors:+ self.cancel_jobs.set()if self.provide_traceback:- logger.error(- f"Error processing item {item}: {e}\nStack trace:\n{traceback.format_exc()}"- )+ logger.error(f"Error for {item}: {e}\n{traceback.format_exc()}")else:logger.error(- f"Error processing item {item}: {e}. Set `provide_traceback=True` to see the stack trace."+ f"Error for {item}: {e}. "+ "Set `provide_traceback=True` for traceback.")return None- return wrapped-- def _execute_isolated_single_thread(self, function, data):- results = []- pbar = tqdm.tqdm(- total=len(data),- dynamic_ncols=True,- disable=self.disable_progress_bar,- file=sys.stdout- )-- from dspy.dsp.utils.settings import thread_local_overrides- original_overrides = thread_local_overrides.overrides- for item in data:- with logging_redirect_tqdm():- if self.cancel_jobs.is_set():- break-- # Create an isolated context for each task by copying current overrides- # This way, even if an iteration modifies the overrides, it won't affect subsequent iterations- thread_local_overrides.overrides = original_overrides.copy()-- try:- result = function(item)- results.append(result)- finally:- thread_local_overrides.overrides = original_overrides-- if self.compare_results:- # Assumes score is the last element of the result tuple- self._update_progress(- pbar,- sum([r[-1] for r in results if r is not None]),- len([r for r in data if r is not None]),- )- else:- self._update_progress(pbar, len(results), len(data))+ return safe_func- pbar.close()+ def _execute_parallel(self, function, data):+ results = [None] * len(data)+ job_cancelled = "cancelled"- if self.cancel_jobs.is_set():- logger.warning("Execution was cancelled due to errors.")- raise Exception("Execution was cancelled due to errors.")+ # We resubmit at most once per item.+ start_time_map = {}+ start_time_lock = threading.Lock()+ resubmitted = set()- return results+ # This is the worker function each thread will run.+ def worker(parent_overrides, submission_id, index, item):+ if self.cancel_jobs.is_set():+ return index, job_cancelled+ # Record actual start time+ with start_time_lock:+ start_time_map[submission_id] = time.time()- def _update_progress(self, pbar, nresults, ntotal):- if self.compare_results:- percentage = round(100 * nresults / ntotal, 1) if ntotal > 0 else 0- pbar.set_description(f"Average Metric: {nresults:.2f} / {ntotal} ({percentage}%)")- else:- pbar.set_description(f"Processed {nresults} / {ntotal} examples")+ # Apply parent's thread-local overrides+ from dspy.dsp.utils.settings import thread_local_overrides- pbar.update()+ original = thread_local_overrides.overrides+ thread_local_overrides.overrides = parent_overrides.copy()- def _execute_multi_thread(self, function, data):- results = [None] * len(data) # Pre-allocate results list to maintain order- job_cancelled = "cancelled"+ try:+ return index, function(item)+ finally:+ thread_local_overrides.overrides = original+ # Handle Ctrl-C in the main thread@contextlib.contextmanager- def interrupt_handler_manager():- """Sets the cancel_jobs event when a SIGINT is received, only in the main thread."""-- # TODO: Is this check conducive to nested usage of ParallelExecutor?+ def interrupt_manager():if threading.current_thread() is threading.main_thread():- default_handler = signal.getsignal(signal.SIGINT)+ orig_handler = signal.getsignal(signal.SIGINT)- def interrupt_handler(sig, frame):+ def handler(sig, frame):self.cancel_jobs.set()- logger.warning("Received SIGINT. Cancelling execution.")- # Re-raise the signal to allow default behavior- default_handler(sig, frame)+ logger.warning("SIGINT received. Cancelling.")+ orig_handler(sig, frame)- signal.signal(signal.SIGINT, interrupt_handler)+ signal.signal(signal.SIGINT, handler)try:yieldfinally:- signal.signal(signal.SIGINT, default_handler)+ signal.signal(signal.SIGINT, orig_handler)else:- # If not in the main thread, skip setting signal handlersyield- def cancellable_function(parent_overrides, index_item):- index, item = index_item- if self.cancel_jobs.is_set():- return index, job_cancelled+ executor = ThreadPoolExecutor(max_workers=self.num_threads)+ try:+ with interrupt_manager():+ from dspy.dsp.utils.settings import thread_local_overrides- # Create an isolated context for each task by copying parent's overrides- from dspy.dsp.utils.settings import thread_local_overrides- original_overrides = thread_local_overrides.overrides- thread_local_overrides.overrides = parent_overrides.copy()+ parent_overrides = thread_local_overrides.overrides.copy()- try:- return index, function(item)- finally:- thread_local_overrides.overrides = original_overrides+ futures_map = {}+ futures_set = set()+ submission_counter = 0- with ThreadPoolExecutor(max_workers=self.num_threads) as executor, interrupt_handler_manager():- from dspy.dsp.utils.settings import thread_local_overrides- parent_overrides = thread_local_overrides.overrides.copy()-- futures = {}- for pair in enumerate(data):- # Pass the parent thread's overrides to each thread- future = executor.submit(cancellable_function, parent_overrides, pair)- futures[future] = pair-- pbar = tqdm.tqdm(- total=len(data),- dynamic_ncols=True,- disable=self.disable_progress_bar,- file=sys.stdout- )-- for future in as_completed(futures):- index, result = future.result()-- if result is job_cancelled:- continue-- results[index] = result-- if self.compare_results:- # Assumes score is the last element of the result tuple- self._update_progress(- pbar,- sum([r[-1] for r in results if r is not None]),- len([r for r in results if r is not None]),+ for idx, item in enumerate(data):+ f = executor.submit(+ worker, parent_overrides, submission_counter, idx, item)- else:- self._update_progress(- pbar,- len([r for r in results if r is not None]),- len(data),+ futures_map[f] = (submission_counter, idx, item)+ futures_set.add(f)+ submission_counter += 1++ pbar = tqdm.tqdm(+ total=len(data),+ dynamic_ncols=True,+ disable=self.disable_progress_bar,+ file=sys.stdout,+ )++ def all_done():+ return all(r is not None for r in results)++ while futures_set and not self.cancel_jobs.is_set():+ if all_done():+ break+ done, not_done = wait(+ futures_set, timeout=1, return_when=FIRST_COMPLETED)-- pbar.close()+ for f in done:+ futures_set.remove(f)+ try:+ index, outcome = f.result()+ except Exception:+ pass+ else:+ if outcome != job_cancelled and results[index] is None:+ results[index] = outcome++ # Update progress+ if self.compare_results:+ vals = [r[-1] for r in results if r is not None]+ self._update_progress(pbar, sum(vals), len(vals))+ else:+ self._update_progress(+ pbar,+ len([r for r in results if r is not None]),+ len(data),+ )++ if all_done():+ break++ # Check stragglers if few remain+ if 0 < self.timeout and len(not_done) <= self.straggler_limit:+ now = time.time()+ for f in list(not_done):+ if f not in resubmitted:+ sid, idx, item = futures_map[f]+ with start_time_lock:+ st = start_time_map.get(sid, None)+ if st and (now - st) >= self.timeout:+ resubmitted.add(f)+ nf = executor.submit(+ worker,+ parent_overrides,+ submission_counter,+ idx,+ item,+ )+ futures_map[nf] = (submission_counter, idx, item)+ futures_set.add(nf)+ submission_counter += 1++ pbar.close()++ finally:+ # Avoid waiting on leftover tasks that no longer matter+ executor.shutdown(wait=False)if self.cancel_jobs.is_set():- logger.warning("Execution was cancelled due to errors.")- raise Exception("Execution was cancelled due to errors.")+ logger.warning("Execution cancelled due to errors or interruption.")+ raise Exception("Execution cancelled due to errors or interruption.")return results++ def _update_progress(self, pbar, nresults, ntotal):+ if self.compare_results:+ pct = round(100 * nresults / ntotal, 1) if ntotal else 0+ pbar.set_description(f"Average Metric: {nresults:.2f} / {ntotal} ({pct}%)")+ else:+ pbar.set_description(f"Processed {nresults} / {ntotal} examples")+ pbar.update()