Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Add CL tool to compare results

Kilian Lieret606d366dd222
diff --git a/sweagent/run/compare_runs.py b/sweagent/run/compare_runs.py
new file mode 100644
index 00000000..076a82af
--- /dev/null
+++ b/sweagent/run/compare_runs.py
@@ -0,0 +1,54 @@
+import argparse
+import json
+from pathlib import Path
+
+
+def get_resolved(path: Path) -> set[str]:
+ data = json.loads(path.read_text())
+ if "resolved" in data:
+ data["resolved_ids"] = data["resolved"]
+ return set(data["resolved_ids"])
+
+
+def get_submitted(path: Path) -> set[str]:
+ return set(json.loads(path.read_text())["submitted_ids"])
+
+
+def compare(new_path, old_path, *, show_same=False):
+ evaluated_ids = get_submitted(new_path)
+ old_evaluated_ids = get_submitted(old_path)
+
+ for id in evaluated_ids:
+ resolved_now = id in get_resolved(new_path)
+ resolved_before = id in get_resolved(old_path)
+ if id not in old_evaluated_ids:
+ emoji = "❓"
+ elif resolved_now and not resolved_before:
+ emoji = "😀"
+ elif resolved_now and resolved_before:
+ emoji = "✅"
+ if not show_same:
+ continue
+ elif not resolved_now and resolved_before:
+ emoji = "❌"
+ else:
+ emoji = "👾"
+ if not show_same:
+ continue
+ print(f"{emoji} {id}")
+
+
+def run_from_cli(_args: list[str] | None = None):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("new_path", type=Path)
+ parser.add_argument("old_path", type=Path)
+ parser.add_argument("--show-same", action="store_true")
+ args = parser.parse_args(_args)
+ print("Emoji legend:")
+ print("❓: Not evaluated in old version")
+ print("😀: Newly resolved in new version")
+ print("✅: Resolved in both")
+ print("❌: Resolved in old, not in new")
+ print("👾: Unresolved in both")
+ print("-" * 80)
+ compare(args.new_path, args.old_path, show_same=args.show_same)
diff --git a/sweagent/run/run.py b/sweagent/run/run.py
index 24637ff6..8e2bcd63 100644
--- a/sweagent/run/run.py
+++ b/sweagent/run/run.py
@@ -50,6 +50,7 @@ def get_cli():
"i",
"I",
"extract-pred",
+ "compare-runs",
],
nargs="?",
)
@@ -112,6 +113,10 @@ def main(args: list[str] | None = None):
from sweagent.run.extract_pred import run_from_cli as extract_pred_main
extract_pred_main(remaining_args)
+ elif command == "compare-runs":
+ from sweagent.run.compare_runs import run_from_cli as compare_runs_main
+
+ compare_runs_main(remaining_args)
else:
msg = f"Unknown command: {command}"
raise ValueError(msg)