SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Allow to specify directories instead of results files
Kilian Lieret43641e5dadd4
diff --git a/sweagent/run/compare_runs.py b/sweagent/run/compare_runs.pyindex 076a82af..2e74696f 100644--- a/sweagent/run/compare_runs.py+++ b/sweagent/run/compare_runs.py@@ -17,10 +17,14 @@ def get_submitted(path: Path) -> set[str]:def compare(new_path, old_path, *, show_same=False):evaluated_ids = get_submitted(new_path)old_evaluated_ids = get_submitted(old_path)+ print(f"Total evaluated: new {len(evaluated_ids)}, old {len(old_evaluated_ids)}")+ resolved_ids = get_resolved(new_path)+ old_resolved_ids = get_resolved(old_path)+ print(f"Total resolved: new {len(resolved_ids)}, old {len(old_resolved_ids)}")for id in evaluated_ids:- resolved_now = id in get_resolved(new_path)- resolved_before = id in get_resolved(old_path)+ resolved_now = id in resolved_ids+ resolved_before = id in old_resolved_idsif id not in old_evaluated_ids:emoji = "❓"elif resolved_now and not resolved_before:@@ -44,6 +48,11 @@ def run_from_cli(_args: list[str] | None = None):parser.add_argument("old_path", type=Path)parser.add_argument("--show-same", action="store_true")args = parser.parse_args(_args)+ if args.new_path.is_dir():+ args.new_path = args.new_path / "results.json"+ if args.old_path.is_dir():+ args.old_path = args.old_path / "results.json"+ print("-" * 80)print("Emoji legend:")print("❓: Not evaluated in old version")print("😀: Newly resolved in new version")