Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Enh(run_batch): save swe-bench reports to results.json

Kilian Lieret19a820104130
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.py
index 1aee8b98..410d8d77 100644
--- a/sweagent/run/hooks/swe_bench_evaluate.py
+++ b/sweagent/run/hooks/swe_bench_evaluate.py
@@ -75,6 +75,19 @@ class SweBenchEvaluate(RunHook):
)
)
+ def move_sb_cli_report(self) -> None:
+ """Move report from `sb-cli-reports` to `results.json`."""
+ output_dir = self.output_dir / "sb-cli-reports"
+ if not output_dir.exists():
+ self.logger.warning("No SweBench report found at %s", output_dir)
+ return
+ (self.output_dir / "results.json").unlink(missing_ok=True)
+ reports = list(output_dir.glob("*.json"))
+ if len(reports) != 1:
+ self.logger.warning("Expected 1 SweBench report at %s, found %d. Cannot rename.", output_dir, len(reports))
+ return
+ reports[0].rename(self.output_dir / "results.json")
+
def on_end(self) -> None:
self.logger.info("Submitting results to SWE-Bench")
try:
@@ -90,3 +103,4 @@ class SweBenchEvaluate(RunHook):
# remove temporary predictions if they exist
if (self.output_dir / "tmppreds.json").exists():
(self.output_dir / "tmppreds.json").unlink()
+ self.move_sb_cli_report()