SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Continuously evaluate SB predictions
Kilian Lieretf7a91a449d9d
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.pyindex 44efb31c..375ee9d3 100644--- a/sweagent/run/hooks/swe_bench_evaluate.py+++ b/sweagent/run/hooks/swe_bench_evaluate.py@@ -1,40 +1,68 @@+import shutilimport subprocessimport sysfrom pathlib import Path+from threading import Lockfrom sweagent.run.hooks.abstract import RunHook+from sweagent.run.merge_predictions import merge_predictions+from sweagent.types import AgentRunResultfrom sweagent.utils.log import get_loggerclass SweBenchEvaluate(RunHook):_SUBSET_MAP = {"lite": "swe-bench_lite"}- def __init__(self, output_dir: Path, subset: str, split: str) -> None:+ def __init__(self, output_dir: Path, subset: str, split: str, continuous_submission_every: int = 0) -> None:super().__init__()self.output_dir = output_dirself.subset = subsetself.split = split+ self.continuous_submission_every = continuous_submission_everyself.logger = get_logger("SB-evaluate", emoji="😬")+ self.n_completed = 0+ self.merge_lock = Lock()++ def _get_sb_call(self, preds_path: Path, prefix: str = "") -> list[str]:+ return [+ "sb-cli",+ "submit",+ self._SUBSET_MAP[self.subset],+ self.split,+ "--predictions_path",+ str(preds_path),+ "--run_id",+ self.output_dir.name,+ "--output_dir",+ str(self.output_dir / f"{prefix}sb-cli-reports"),+ ]++ def on_instance_completed(self, *, result: AgentRunResult):+ self.n_completed += 1+ if self.continuous_submission_every == 0:+ return+ if self.n_completed % self.continuous_submission_every != self.continuous_submission_every - 1:+ return+ with self.merge_lock:+ merge_predictions([self.output_dir], self.output_dir / "tmppreds.json")+ subprocess.Popen(+ self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", prefix="-tmp"),+ )def on_end(self) -> None:self.logger.info("Submitting results to SWE-Bench")try:subprocess.run(- [- "sb-cli",- "submit",- self._SUBSET_MAP[self.subset],- self.split,- "--predictions_path",- self.output_dir / "preds.json",- "--run_id",- self.output_dir.name,- "--output_dir",- self.output_dir / "sb-cli-reports",- ],+ self._get_sb_call(preds_path=self.output_dir / "preds.json"),check=True,stdout=sys.stdout,stderr=sys.stderr,)except subprocess.CalledProcessError as e:self.logger.error("Failed to submit results to SweBench eval: %s", e)+ else:+ # remove temporary predictions if they exist+ if (self.output_dir / "tmppreds.json").exists():+ (self.output_dir / "tmppreds.json").unlink()+ if (self.output_dir / "tmp-sb-cli-reports").exists():+ shutil.rmtree(self.output_dir / "tmp-sb-cli-reports")diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.pyindex fa28cb6a..a4a08896 100644--- a/sweagent/run/run_batch.py+++ b/sweagent/run/run_batch.py@@ -188,7 +188,10 @@ class RunBatch:rb.add_hook(SweBenchEvaluate(- output_dir=config.output_dir, subset=config.instances.subset, split=config.instances.split+ output_dir=config.output_dir,+ subset=config.instances.subset,+ split=config.instances.split,+ continuous_submission_every=5,))return rb