Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

run_batch: Do not wait for sb-cli report completion

Kilian Lieretb71ab550553b
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.py
index 326e8971..83c833e7 100644
--- a/sweagent/run/hooks/swe_bench_evaluate.py
+++ b/sweagent/run/hooks/swe_bench_evaluate.py
@@ -25,7 +25,7 @@ class SweBenchEvaluate(RunHook):
self.last_evaluation_time = time()
self.evaluation_interval = continuous_submission_every
- def _get_sb_call(self, preds_path: Path, prefix: str = "", overwrite: bool = False) -> list[str]:
+ def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:
args = [
"sb-cli",
"submit",
@@ -36,10 +36,10 @@ class SweBenchEvaluate(RunHook):
"--run_id",
self.output_dir.name,
"--output_dir",
- str(self.output_dir / f"{prefix}sb-cli-reports"),
+ str(self.output_dir / "sb-cli-reports"),
]
- if overwrite:
- args.append("--overwrite")
+ if submit_only:
+ args.extend(["--wait_for_evaluation", "0", "--gen_report", "0", "--verify_submission", "0"])
return args
def on_instance_completed(self, *, result: AgentRunResult):
@@ -55,7 +55,7 @@ class SweBenchEvaluate(RunHook):
self.last_evaluation_time = current_time
subprocess.Popen(
- self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", prefix="tmp-", overwrite=True),
+ self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", submit_only=True),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.py
index dc69af0e..46b8c5f2 100644
--- a/sweagent/run/run_batch.py
+++ b/sweagent/run/run_batch.py
@@ -191,7 +191,7 @@ class RunBatch:
output_dir=config.output_dir,
subset=config.instances.subset,
split=config.instances.split,
- continuous_submission_every=5 * 60,
+ continuous_submission_every=30,
)
)
return rb