SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
run_batch: Do not wait for sb-cli report completion
Kilian Lieretb71ab550553b
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.pyindex 326e8971..83c833e7 100644--- a/sweagent/run/hooks/swe_bench_evaluate.py+++ b/sweagent/run/hooks/swe_bench_evaluate.py@@ -25,7 +25,7 @@ class SweBenchEvaluate(RunHook):self.last_evaluation_time = time()self.evaluation_interval = continuous_submission_every- def _get_sb_call(self, preds_path: Path, prefix: str = "", overwrite: bool = False) -> list[str]:+ def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:args = ["sb-cli","submit",@@ -36,10 +36,10 @@ class SweBenchEvaluate(RunHook):"--run_id",self.output_dir.name,"--output_dir",- str(self.output_dir / f"{prefix}sb-cli-reports"),+ str(self.output_dir / "sb-cli-reports"),]- if overwrite:- args.append("--overwrite")+ if submit_only:+ args.extend(["--wait_for_evaluation", "0", "--gen_report", "0", "--verify_submission", "0"])return argsdef on_instance_completed(self, *, result: AgentRunResult):@@ -55,7 +55,7 @@ class SweBenchEvaluate(RunHook):self.last_evaluation_time = current_timesubprocess.Popen(- self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", prefix="tmp-", overwrite=True),+ self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", submit_only=True),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,)diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.pyindex dc69af0e..46b8c5f2 100644--- a/sweagent/run/run_batch.py+++ b/sweagent/run/run_batch.py@@ -191,7 +191,7 @@ class RunBatch:output_dir=config.output_dir,subset=config.instances.subset,split=config.instances.split,- continuous_submission_every=5 * 60,+ continuous_submission_every=30,))return rb