Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

CI: Test fixes and skips

Kilian Lieret291683f4bbf2
diff --git a/tests/test_agent.py b/tests/test_agent.py
index 6a942ac2..a07e9070 100644
--- a/tests/test_agent.py
+++ b/tests/test_agent.py
@@ -186,13 +186,16 @@ def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: Age
assert a.info["exit_status"] == "exit_cost" # type: ignore
+# todo: fixme; Needs real environment or mocking of read_file
def test_run_autosubmit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["raise_cost"]) # type: ignore
a.setup(dummy_env, EmptyProblemStatement())
+ dummy_env.write_file("/root/model.patch", "mysubmission")
dummy_env.deployment.runtime.run_in_session_outputs = [ # type: ignore
BashObservation(output=""),
- BashObservation(output=r"<<SUBMISSION||mysubmission||SUBMISSION>>"),
+ BashObservation(output=r"<<SWE_AGENT_SUBMISSION>>\nmysubmission\n<<SWE_AGENT_SUBMISSION>>"),
]
r = a.step()
assert a.info is not None
@@ -216,11 +219,14 @@ def test_show_no_output_template(dummy_env: SWEEnv, default_agent: Agent, tmp_pa
# todo: actually test that the template is used
+# todo: fixme; Needs real environment or mocking of read_file
def test_successful_submission(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["```\nsubmit\n```"]) # type: ignore
a.setup(dummy_env, EmptyProblemStatement())
- dummy_env.deployment.runtime.run_in_session_outputs = BashObservation(output=r"<<SUBMISSION||test||SUBMISSION>>") # type: ignore
+ dummy_env.write_file("/root/model.patch", "test")
+ dummy_env.deployment.runtime.run_in_session_outputs = BashObservation(output=r"<<SWE_AGENT_SUBMISSION>>") # type: ignore
a.step()
assert a.info["exit_status"] == "submitted" # type: ignore
assert a.info["submission"] == "test" # type: ignore
diff --git a/tests/test_batch_instance.py b/tests/test_batch_instance.py
index c1184f85..08c33020 100644
--- a/tests/test_batch_instance.py
+++ b/tests/test_batch_instance.py
@@ -16,7 +16,7 @@ def test_simple_batch_from_swe_bench_to_full_batch_instance(test_data_sources_pa
assert isinstance(instance.env.repo, PreExistingRepoConfig)
assert instance.env.repo.repo_name == "testbed"
assert isinstance(instance.env.deployment, DockerDeploymentConfig)
- assert instance.env.deployment.image == "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1458:latest"
+ assert instance.env.deployment.image == "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1458:v1"
assert isinstance(instance.problem_statement, TextProblemStatement)
assert instance.problem_statement.text == sb_instance["problem_statement"]
assert instance.problem_statement.id == "pydicom__pydicom-1458"
diff --git a/tests/test_env.py b/tests/test_env.py
index 9b96f076..41e2a16b 100644
--- a/tests/test_env.py
+++ b/tests/test_env.py
@@ -32,6 +32,7 @@ def test_startup_commands(test_env_args):
pass
@pytest.mark.slow
def test_read_file(tmp_path, test_env_args):
with swe_env_context(test_env_args) as env:
@@ -49,14 +50,14 @@ def test_env_with_hook(test_env_args):
@pytest.mark.slow
def test_env_communicate_with_handling(test_env_args):
with swe_env_context(test_env_args) as env:
- env.communicate("echo 'hello world'", check=True, error_msg="Failed to echo")
+ env.communicate("echo 'hello world'", check="raise", error_msg="Failed to echo")
@pytest.mark.slow
def test_env_communicate_with_handling_timeout(test_env_args):
with swe_env_context(test_env_args) as env:
with pytest.raises(CommandTimeoutError):
- env.communicate("sleep 10", check=True, error_msg="Failed to sleep", timeout=0.2)
+ env.communicate("sleep 10", check="raise", error_msg="Failed to sleep", timeout=0.2)
@pytest.mark.slow