SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Revert "Feat: Basic enigma/ctf support (#1026)"
Kilian Lieret392651976b4b
diff --git a/sweagent/agent/problem_statement.py b/sweagent/agent/problem_statement.pyindex 07a4ef48..897d7724 100644--- a/sweagent/agent/problem_statement.py+++ b/sweagent/agent/problem_statement.py@@ -5,7 +5,6 @@ from pathlib import Pathfrom typing import Any, Literal, Protocolfrom pydantic import BaseModel, ConfigDict, Field-from pydantic_core import from_jsonfrom sweagent.utils.github import _get_problem_statement_from_github_issue, _parse_gh_issue_urlfrom sweagent.utils.log import get_logger@@ -13,13 +12,6 @@ from sweagent.utils.log import get_loggerlogger = get_logger("swea-config", emoji="🔧")-def _get_ctf_json(data: dict[str, Any]) -> dict[str, Any]:- try:- return from_json(data["path"].read_text())- except Exception:- return dict()--class ProblemStatement(Protocol):"""A problem statement for a task."""@@ -133,50 +125,11 @@ class GithubIssue(BaseModel):return self.extra_fields-class CTFProblemStatement(BaseModel):- path: Path-- json_data: dict[str, Any] = Field(- default_factory=_get_ctf_json,- frozen=True,- exclude=True,- )- name: str = Field(default_factory=lambda data: data["json_data"].get("name"))- category: Literal["crypto", "rev", "web", "forensics", "pwn", "misc"] = Field(- default_factory=lambda data: data["json_data"].get("category")- )- description: str = Field(default_factory=lambda data: data["json_data"].get("description"))- files: list[str] = Field(default_factory=lambda data: data["json_data"].get("files"))- flag: str = Field(default_factory=lambda data: data["json_data"].get("flag"))-- extra_fields: dict[str, Any] = Field(default_factory=dict)- """Any additional data to be added to the instance.- This data will be available when formatting prompt templates.- """-- type: Literal["ctf_json"] = "ctf_json"- """Discriminator for (de)serialization/CLI. Do not change."""-- id: str = Field(default_factory=lambda data: "_".join([str(data["category"]), str(data["name"])]))-- model_config = ConfigDict(extra="forbid")-- def get_problem_statement(self) -> str:- return self.description-- def get_extra_fields(self) -> dict[str, Any]:- extra_fields = self.model_dump()- extra_fields.update(self.extra_fields)- return extra_fields---ProblemStatementConfig = (- TextProblemStatement | GithubIssue | EmptyProblemStatement | FileProblemStatement | CTFProblemStatement-)+ProblemStatementConfig = TextProblemStatement | GithubIssue | EmptyProblemStatement | FileProblemStatementdef problem_statement_from_simplified_input(- *, input: str, type: Literal["text", "text_file", "github_issue", "ctf_json"]+ *, input: str, type: Literal["text", "text_file", "github_issue"]) -> ProblemStatementConfig:"""Get a problem statement from an `input` string and a `type`.@@ -190,8 +143,6 @@ def problem_statement_from_simplified_input(return FileProblemStatement(path=Path(input))elif type == "github_issue":return GithubIssue(github_url=input)- elif type == "ctf_json":- return CTFProblemStatement(path=Path(input))else:msg = f"Unknown problem statement type: {type}"raise ValueError(msg)diff --git a/sweagent/environment/repo.py b/sweagent/environment/repo.pyindex efcef7e2..0abf4438 100644--- a/sweagent/environment/repo.py+++ b/sweagent/environment/repo.py@@ -6,7 +6,6 @@ from typing import Any, Literal, Protocolfrom git import InvalidGitRepositoryErrorfrom git import Repo as GitRepofrom pydantic import BaseModel, ConfigDict, Field-from pydantic_core import from_jsonfrom swerex.deployment.abstract import AbstractDeploymentfrom swerex.runtime.abstract import Command, UploadRequestfrom typing_extensions import Self@@ -182,53 +181,11 @@ class GithubRepoConfig(BaseModel):return _get_git_reset_commands(self.base_commit)-class CTFRepoConfig(BaseModel):- path: Path- challenge_json_filename: str = Field(default="challenge.json")- """The JSON file containing the CTF challenge.- This filename should be a relative path inside the repository parent path.- """-- base_commit: str = Field(default="HEAD")- """This field has no effect on CTF repo."""-- type: Literal["ctf"]- """Discriminator for (de)serialization/CLI. Do not change."""-- model_config = ConfigDict(extra="forbid")-- @property- def repo_name(self) -> str:- """Set automatically based on the repository name. Cannot be set."""- return Path(self.path).resolve().name.replace(" ", "-").replace("'", "")-- @property- def files(self) -> list[str]:- json_data = (self.path / self.challenge_json_filename).read_text()- return from_json(json_data)["files"]-- def copy(self, deployment: AbstractDeployment):- for file in self.files:- asyncio.run(- deployment.runtime.upload(- UploadRequest(source_path=str(self.path / Path(file)), target_path=f"/{self.repo_name}/{file}")- )- )- r = asyncio.run(deployment.runtime.execute(Command(command=f"chown -R root:root {self.repo_name}", shell=True)))- if r.exit_code != 0:- msg = f"Failed to change permissions on copied repository (exit code: {r.exit_code}, stdout: {r.stdout}, stderr: {r.stderr})"- raise RuntimeError(msg)-- def get_reset_commands(self) -> list[str]:- """Issued after the copy operation or when the environment is reset."""- return ["git init"]---RepoConfig = LocalRepoConfig | GithubRepoConfig | PreExistingRepoConfig | CTFRepoConfig+RepoConfig = LocalRepoConfig | GithubRepoConfig | PreExistingRepoConfigdef repo_from_simplified_input(- *, input: str, base_commit: str = "HEAD", type: Literal["local", "github", "preexisting", "auto", "ctf"] = "auto"+ *, input: str, base_commit: str = "HEAD", type: Literal["local", "github", "preexisting", "auto"] = "auto") -> RepoConfig:"""Get repo config from a simplified input.@@ -248,7 +205,5 @@ def repo_from_simplified_input(return GithubRepoConfig(github_url=input, base_commit=base_commit)else:return LocalRepoConfig(path=Path(input), base_commit=base_commit)- if type == "ctf":- return CTFRepoConfig(path=Path(input))msg = f"Unknown repo type: {type}"raise ValueError(msg)diff --git a/sweagent/run/run_single.py b/sweagent/run/run_single.pyindex bf528774..8ba4f77b 100644--- a/sweagent/run/run_single.py+++ b/sweagent/run/run_single.py@@ -33,17 +33,15 @@ from pathlib import Pathfrom typing import Selfimport yaml-from pydantic import BaseModel, ConfigDict, Field, model_validator+from pydantic import BaseModel, ConfigDict, Fieldfrom pydantic_settings import BaseSettings, SettingsConfigDictfrom sweagent.agent.agents import AbstractAgent, AgentConfig, get_agent_from_configfrom sweagent.agent.problem_statement import (- CTFProblemStatement,EmptyProblemStatement,ProblemStatement,ProblemStatementConfig,)-from sweagent.environment.repo import CTFRepoConfigfrom sweagent.environment.swe_env import EnvironmentConfig, SWEEnvfrom sweagent.run.common import AutoCorrectSuggestion as ACSfrom sweagent.run.common import BasicCLI, ConfigHelper, save_predictions@@ -118,12 +116,6 @@ class RunSingleConfig(BaseSettings, cli_implicit_flags=False):ACS("repo.path", "env.repo.path"),]- @model_validator(mode="after")- def check_repo_and_problem_statement_config(self) -> Self:- if isinstance(self.problem_statement, CTFProblemStatement) != isinstance(self.env.repo, CTFRepoConfig):- ValueError("CTF problem statement should be used only with CTF repo config!")- return self-class RunSingle:def __init__(diff --git a/tests/conftest.py b/tests/conftest.pyindex fabd72e8..b1314fa3 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -50,14 +50,6 @@ def ctf_data_path(test_data_sources_path) -> Path:return p-def swe_agent_test_ctf_repo_and_ps(ctf_data_path) -> tuple[Path, Path]:- p = ctf_data_path / "forensics" / "flash"- ps = p / "challenge.json"- assert p.is_dir()- return p, ps--@pytest.fixturedef test_data_sources_path(test_data_path) -> Path:p = test_data_path / "data_sources"diff --git a/tests/test_run_single.py b/tests/test_run_single.pyindex 185b08a2..59e8fff0 100644--- a/tests/test_run_single.py+++ b/tests/test_run_single.py@@ -123,33 +123,3 @@ def test_run_ies_repo_ps_matrix(for fmt in output_formats:assert len(list(Path(tmpdir).rglob(f"*.{fmt}"))) == 1print(fmt, list(Path(tmpdir).iterdir()))---def test_run_single_ctf(tmpdir, swe_agent_test_ctf_repo_and_ps):- # ies = instant empty submit- output_formats = ["traj", "pred", "patch"]- for fmt in output_formats:- assert not list(Path(tmpdir).glob(f"*.{fmt}"))- repo, ps = swe_agent_test_ctf_repo_and_ps- ps_args = ["--problem_statement.path", str(ps), "--problem_statement.type", "ctf_json"]- repo_args = ["--env.repo.path", str(repo), "--env.repo.type", "ctf"]- args = [- "--agent.model.name=instant_empty_submit",- "--output_dir",- str(tmpdir),- *ps_args,- *repo_args,- "--config",- str(CONFIG_DIR / "default_no_fcalls.yaml"),- ]- print(args)- rs_config = BasicCLI(RunSingleConfig).get_config(args)- print(rs_config)- rs = RunSingle.from_config(rs_config) # type: ignore- with tmpdir.as_cwd():- # Test that we can run run.py also independently from repo dir- rs.run()- for fmt in output_formats:- assert len(list(Path(tmpdir).rglob(f"*.{fmt}"))) == 1- print(fmt, list(Path(tmpdir).iterdir()))