SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Enh: Warn if sb-cli failed
Kilian Lieretd99481b417f0
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.pyindex 83c833e7..1aee8b98 100644--- a/sweagent/run/hooks/swe_bench_evaluate.py+++ b/sweagent/run/hooks/swe_bench_evaluate.py@@ -1,4 +1,8 @@-import shutil+"""SweBench evaluation hook.++Will be automatically added to `run_batch` if `SWEBenchInstances.evaluate` is set to true+"""+import subprocessimport sysfrom pathlib import Path@@ -24,6 +28,7 @@ class SweBenchEvaluate(RunHook):self.merge_lock = Lock()self.last_evaluation_time = time()self.evaluation_interval = continuous_submission_every+ self._running_calls = []def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:args = [@@ -42,6 +47,14 @@ class SweBenchEvaluate(RunHook):args.extend(["--wait_for_evaluation", "0", "--gen_report", "0", "--verify_submission", "0"])return args+ def check_running_calls(self) -> None:+ """Warn if one of the running calls failed."""+ for call in self._running_calls:+ if call.poll() is not None:+ if call.returncode != 0:+ self.logger.error("Failed to submit results to SweBench eval: %s", call.stderr.read())+ self._running_calls.remove(call)+def on_instance_completed(self, *, result: AgentRunResult):if self.evaluation_interval == 0:return@@ -54,10 +67,12 @@ class SweBenchEvaluate(RunHook):merge_predictions([self.output_dir], self.output_dir / "tmppreds.json")self.last_evaluation_time = current_time- subprocess.Popen(- self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", submit_only=True),- stdout=subprocess.DEVNULL,- stderr=subprocess.DEVNULL,+ self._running_calls.append(+ subprocess.Popen(+ self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", submit_only=True),+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ ))def on_end(self) -> None:@@ -75,5 +90,3 @@ class SweBenchEvaluate(RunHook):# remove temporary predictions if they existif (self.output_dir / "tmppreds.json").exists():(self.output_dir / "tmppreds.json").unlink()- if (self.output_dir / "tmp-sb-cli-reports").exists():- shutil.rmtree(self.output_dir / "tmp-sb-cli-reports")