SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
fix: map multimodal subset to sb-cli's swe-bench-m (#1458)
Anas Khan3ea751c087f3
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.pyindex 09498d97..38b806db 100644--- a/sweagent/run/hooks/swe_bench_evaluate.py+++ b/sweagent/run/hooks/swe_bench_evaluate.py@@ -17,7 +17,10 @@ from sweagent.utils.log import get_loggerclass SweBenchEvaluate(RunHook):- _SUBSET_MAP = {"lite": "swe-bench_lite", "verified": "swe-bench_verified", "multimodal": "swe-bench_multimodal"}+ # Maps SWEBenchInstances.subset values to the subset identifiers accepted by+ # sb-cli. sb-cli only supports these three; "full" and "multilingual" have no+ # sb-cli equivalent and therefore cannot be evaluated this way.+ _SUBSET_MAP = {"lite": "swe-bench_lite", "verified": "swe-bench_verified", "multimodal": "swe-bench-m"}def __init__(self, output_dir: Path, subset: str, split: str, continuous_submission_every: int = 0) -> None:super().__init__()@@ -38,6 +41,12 @@ class SweBenchEvaluate(RunHook):return f"{self.output_dir.name}_{self._time_suffix}"def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:+ if self.subset not in self._SUBSET_MAP:+ msg = (+ f"Cannot evaluate subset {self.subset!r} with sb-cli. "+ f"Supported subsets are: {', '.join(self._SUBSET_MAP)}."+ )+ raise ValueError(msg)args = ["sb-cli","submit",diff --git a/tests/test_run_hooks.py b/tests/test_run_hooks.pyindex a1f25ed2..83004c19 100644--- a/tests/test_run_hooks.py+++ b/tests/test_run_hooks.py@@ -8,6 +8,7 @@ import pytestfrom sweagent.agent.problem_statement import GithubIssue, TextProblemStatementfrom sweagent.run.hooks.apply_patch import SaveApplyPatchHookfrom sweagent.run.hooks.open_pr import OpenPRConfig, OpenPRHook+from sweagent.run.hooks.swe_bench_evaluate import SweBenchEvaluatefrom sweagent.types import AgentRunResult@@ -127,3 +128,38 @@ def test_save_apply_patch_hook_concurrent_workers_save_to_correct_dirs(tmp_path)assert patch_b.exists(), "Patch for instance-B was not saved to its own directory"assert patch_a.read_text() == "patch A content"assert patch_b.read_text() == "patch B content"+++ ("subset", "expected"),+ [+ ("lite", "swe-bench_lite"),+ ("verified", "swe-bench_verified"),+ ("multimodal", "swe-bench-m"),+ ],+)+def test_swe_bench_evaluate_subset_matches_sb_cli(tmp_path, subset, expected):+ """The subset passed to sb-cli must be one of its accepted values.++ sb-cli's ``Subset`` enum only accepts ``swe-bench_lite``, ``swe-bench_verified``+ and ``swe-bench-m``. In particular the multimodal dataset is ``swe-bench-m``,+ not ``swe-bench_multimodal``, so submitting the latter is rejected at the CLI+ boundary and ``--evaluate=True`` fails on a multimodal run.+ """+ hook = SweBenchEvaluate(output_dir=tmp_path, subset=subset, split="dev")+ call = hook._get_sb_call(tmp_path / "preds.json")+ # ["sb-cli", "submit", <subset>, <split>, ...]+ assert call[2] == expected++[email protected]("subset", ["full", "multilingual"])+def test_swe_bench_evaluate_unsupported_subset_raises_value_error(tmp_path, subset):+ """Subsets with no sb-cli equivalent must fail with a clear error.++ ``full`` and ``multilingual`` are valid ``SWEBenchInstances.subset`` values for+ loading instances, but sb-cli cannot evaluate them. Building the call must raise+ a descriptive ValueError instead of a bare KeyError.+ """+ hook = SweBenchEvaluate(output_dir=tmp_path, subset=subset, split="dev")+ with pytest.raises(ValueError, match=subset):+ hook._get_sb_call(tmp_path / "preds.json")