SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Add CL tool to compare results
Kilian Lieret606d366dd222
diff --git a/sweagent/run/compare_runs.py b/sweagent/run/compare_runs.pynew file mode 100644index 00000000..076a82af--- /dev/null+++ b/sweagent/run/compare_runs.py@@ -0,0 +1,54 @@+import argparse+import json+from pathlib import Path+++def get_resolved(path: Path) -> set[str]:+ data = json.loads(path.read_text())+ if "resolved" in data:+ data["resolved_ids"] = data["resolved"]+ return set(data["resolved_ids"])+++def get_submitted(path: Path) -> set[str]:+ return set(json.loads(path.read_text())["submitted_ids"])+++def compare(new_path, old_path, *, show_same=False):+ evaluated_ids = get_submitted(new_path)+ old_evaluated_ids = get_submitted(old_path)++ for id in evaluated_ids:+ resolved_now = id in get_resolved(new_path)+ resolved_before = id in get_resolved(old_path)+ if id not in old_evaluated_ids:+ emoji = "❓"+ elif resolved_now and not resolved_before:+ emoji = "😀"+ elif resolved_now and resolved_before:+ emoji = "✅"+ if not show_same:+ continue+ elif not resolved_now and resolved_before:+ emoji = "❌"+ else:+ emoji = "👾"+ if not show_same:+ continue+ print(f"{emoji} {id}")+++def run_from_cli(_args: list[str] | None = None):+ parser = argparse.ArgumentParser()+ parser.add_argument("new_path", type=Path)+ parser.add_argument("old_path", type=Path)+ parser.add_argument("--show-same", action="store_true")+ args = parser.parse_args(_args)+ print("Emoji legend:")+ print("❓: Not evaluated in old version")+ print("😀: Newly resolved in new version")+ print("✅: Resolved in both")+ print("❌: Resolved in old, not in new")+ print("👾: Unresolved in both")+ print("-" * 80)+ compare(args.new_path, args.old_path, show_same=args.show_same)diff --git a/sweagent/run/run.py b/sweagent/run/run.pyindex 24637ff6..8e2bcd63 100644--- a/sweagent/run/run.py+++ b/sweagent/run/run.py@@ -50,6 +50,7 @@ def get_cli():"i","I","extract-pred",+ "compare-runs",],nargs="?",)@@ -112,6 +113,10 @@ def main(args: list[str] | None = None):from sweagent.run.extract_pred import run_from_cli as extract_pred_mainextract_pred_main(remaining_args)+ elif command == "compare-runs":+ from sweagent.run.compare_runs import run_from_cli as compare_runs_main++ compare_runs_main(remaining_args)else:msg = f"Unknown command: {command}"raise ValueError(msg)