Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

fix: map multimodal subset to sb-cli's swe-bench-m (#1458)

Anas Khan3ea751c087f3
diff --git a/sweagent/run/hooks/swe_bench_evaluate.py b/sweagent/run/hooks/swe_bench_evaluate.py
index 09498d97..38b806db 100644
--- a/sweagent/run/hooks/swe_bench_evaluate.py
+++ b/sweagent/run/hooks/swe_bench_evaluate.py
@@ -17,7 +17,10 @@ from sweagent.utils.log import get_logger
class SweBenchEvaluate(RunHook):
- _SUBSET_MAP = {"lite": "swe-bench_lite", "verified": "swe-bench_verified", "multimodal": "swe-bench_multimodal"}
+ # Maps SWEBenchInstances.subset values to the subset identifiers accepted by
+ # sb-cli. sb-cli only supports these three; "full" and "multilingual" have no
+ # sb-cli equivalent and therefore cannot be evaluated this way.
+ _SUBSET_MAP = {"lite": "swe-bench_lite", "verified": "swe-bench_verified", "multimodal": "swe-bench-m"}
def __init__(self, output_dir: Path, subset: str, split: str, continuous_submission_every: int = 0) -> None:
super().__init__()
@@ -38,6 +41,12 @@ class SweBenchEvaluate(RunHook):
return f"{self.output_dir.name}_{self._time_suffix}"
def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:
+ if self.subset not in self._SUBSET_MAP:
+ msg = (
+ f"Cannot evaluate subset {self.subset!r} with sb-cli. "
+ f"Supported subsets are: {', '.join(self._SUBSET_MAP)}."
+ )
+ raise ValueError(msg)
args = [
"sb-cli",
"submit",
diff --git a/tests/test_run_hooks.py b/tests/test_run_hooks.py
index a1f25ed2..83004c19 100644
--- a/tests/test_run_hooks.py
+++ b/tests/test_run_hooks.py
@@ -8,6 +8,7 @@ import pytest
from sweagent.agent.problem_statement import GithubIssue, TextProblemStatement
from sweagent.run.hooks.apply_patch import SaveApplyPatchHook
from sweagent.run.hooks.open_pr import OpenPRConfig, OpenPRHook
+from sweagent.run.hooks.swe_bench_evaluate import SweBenchEvaluate
from sweagent.types import AgentRunResult
@@ -127,3 +128,38 @@ def test_save_apply_patch_hook_concurrent_workers_save_to_correct_dirs(tmp_path)
assert patch_b.exists(), "Patch for instance-B was not saved to its own directory"
assert patch_a.read_text() == "patch A content"
assert patch_b.read_text() == "patch B content"
+
+
+ ("subset", "expected"),
+ [
+ ("lite", "swe-bench_lite"),
+ ("verified", "swe-bench_verified"),
+ ("multimodal", "swe-bench-m"),
+ ],
+)
+def test_swe_bench_evaluate_subset_matches_sb_cli(tmp_path, subset, expected):
+ """The subset passed to sb-cli must be one of its accepted values.
+
+ sb-cli's ``Subset`` enum only accepts ``swe-bench_lite``, ``swe-bench_verified``
+ and ``swe-bench-m``. In particular the multimodal dataset is ``swe-bench-m``,
+ not ``swe-bench_multimodal``, so submitting the latter is rejected at the CLI
+ boundary and ``--evaluate=True`` fails on a multimodal run.
+ """
+ hook = SweBenchEvaluate(output_dir=tmp_path, subset=subset, split="dev")
+ call = hook._get_sb_call(tmp_path / "preds.json")
+ # ["sb-cli", "submit", <subset>, <split>, ...]
+ assert call[2] == expected
+
+
[email protected]("subset", ["full", "multilingual"])
+def test_swe_bench_evaluate_unsupported_subset_raises_value_error(tmp_path, subset):
+ """Subsets with no sb-cli equivalent must fail with a clear error.
+
+ ``full`` and ``multilingual`` are valid ``SWEBenchInstances.subset`` values for
+ loading instances, but sb-cli cannot evaluate them. Building the call must raise
+ a descriptive ValueError instead of a bare KeyError.
+ """
+ hook = SweBenchEvaluate(output_dir=tmp_path, subset=subset, split="dev")
+ with pytest.raises(ValueError, match=subset):
+ hook._get_sb_call(tmp_path / "preds.json")