SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Fix(run-batch): Only wait for first batch of workers
Kilian Lieret7436390e3670
diff --git a/sweagent/run/_progress.py b/sweagent/run/_progress.pyindex 84751a42..945966e2 100644--- a/sweagent/run/_progress.py+++ b/sweagent/run/_progress.py@@ -78,6 +78,10 @@ class RunBatchProgressManager:self.render_group = Group(Table(), self._task_progress_bar, self._main_progress_bar)self._yaml_report_path = yaml_report_path+ @property+ def n_completed(self) -> int:+ return sum(len(instances) for instances in self._instances_by_exit_status.values())+def update_exit_status_table(self):# We cannot update the existing table, so we need to create a new one and# assign it back to the render group.diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.pyindex ed7a68f0..d95ba672 100644--- a/sweagent/run/run_batch.py+++ b/sweagent/run/run_batch.py@@ -84,7 +84,7 @@ class RunBatchConfig(BaseSettings, cli_implicit_flags=False):"""Path to a .env file to load environment variables from."""num_workers: int = Field(default=1)"""Number of parallel workers to use."""- random_delay_multiplier: float = 1+ random_delay_multiplier: float = 0.3"""We will wait for a random amount of time between 0 and `random_delay_multiplier`times the number of workers at the start of each instance. This is to avoid anypotential race conditions.@@ -140,6 +140,7 @@ class RunBatch:redo_existing: bool = False,num_workers: int = 1,progress_bar: bool = True,+ random_delay_multiplier: float = 0.3,):"""Note: When initializing this class, make sure to add the hooks that are required by your actions.See `from_config` for an example.@@ -149,6 +150,9 @@ class RunBatch:num_workers: Number of parallel workers to use. Default is 1 (sequential execution).progress_bar: Whether to show a progress bar. Progress bar is never shown for human models.Progress bar is always shown for multi-worker runs.+ random_delay_multiplier: We will wait for a random amount of time between 0 and `random_delay_multiplier`+ times the number of workers at the start of each instance. This is to avoid any+ potential race conditions."""if self._model_id in ["human", "human_thought"] and num_workers > 1:msg = "Cannot run with human model in parallel"@@ -173,6 +177,7 @@ class RunBatch:num_instances=len(instances), yaml_report_path=output_dir / "run_batch_exit_statuses.yaml")self._show_progress_bar = progress_bar+ self._random_delay_multiplier = random_delay_multiplier@propertydef _model_id(self) -> str:@@ -207,6 +212,7 @@ class RunBatch:redo_existing=config.redo_existing,num_workers=config.num_workers,progress_bar=config.progress_bar,+ random_delay_multiplier=config.random_delay_multiplier,)if isinstance(config.instances, SWEBenchInstances) and config.instances.evaluate:from sweagent.run.hooks.swe_bench_evaluate import SweBenchEvaluate@@ -281,7 +287,8 @@ class RunBatch:register_thread_name(instance.problem_statement.id)self._add_instance_log_file_handlers(instance.problem_statement.id, multi_worker=self._num_workers > 1)# Let's add some randomness to avoid any potential race conditions or thundering herd- time.sleep(random.random() * 0.3 * (self._num_workers - 1))+ if self._progress_manager.n_completed < self._num_workers:+ time.sleep(random.random() * self._random_delay_multiplier * (self._num_workers - 1))self._progress_manager.on_instance_start(instance.problem_statement.id)