Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Revert "Feat: Basic enigma/ctf support (#1026)"

Kilian Lieret392651976b4b
diff --git a/sweagent/agent/problem_statement.py b/sweagent/agent/problem_statement.py
index 07a4ef48..897d7724 100644
--- a/sweagent/agent/problem_statement.py
+++ b/sweagent/agent/problem_statement.py
@@ -5,7 +5,6 @@ from pathlib import Path
from typing import Any, Literal, Protocol
from pydantic import BaseModel, ConfigDict, Field
-from pydantic_core import from_json
from sweagent.utils.github import _get_problem_statement_from_github_issue, _parse_gh_issue_url
from sweagent.utils.log import get_logger
@@ -13,13 +12,6 @@ from sweagent.utils.log import get_logger
logger = get_logger("swea-config", emoji="🔧")
-def _get_ctf_json(data: dict[str, Any]) -> dict[str, Any]:
- try:
- return from_json(data["path"].read_text())
- except Exception:
- return dict()
-
-
class ProblemStatement(Protocol):
"""A problem statement for a task."""
@@ -133,50 +125,11 @@ class GithubIssue(BaseModel):
return self.extra_fields
-class CTFProblemStatement(BaseModel):
- path: Path
-
- json_data: dict[str, Any] = Field(
- default_factory=_get_ctf_json,
- frozen=True,
- exclude=True,
- )
- name: str = Field(default_factory=lambda data: data["json_data"].get("name"))
- category: Literal["crypto", "rev", "web", "forensics", "pwn", "misc"] = Field(
- default_factory=lambda data: data["json_data"].get("category")
- )
- description: str = Field(default_factory=lambda data: data["json_data"].get("description"))
- files: list[str] = Field(default_factory=lambda data: data["json_data"].get("files"))
- flag: str = Field(default_factory=lambda data: data["json_data"].get("flag"))
-
- extra_fields: dict[str, Any] = Field(default_factory=dict)
- """Any additional data to be added to the instance.
- This data will be available when formatting prompt templates.
- """
-
- type: Literal["ctf_json"] = "ctf_json"
- """Discriminator for (de)serialization/CLI. Do not change."""
-
- id: str = Field(default_factory=lambda data: "_".join([str(data["category"]), str(data["name"])]))
-
- model_config = ConfigDict(extra="forbid")
-
- def get_problem_statement(self) -> str:
- return self.description
-
- def get_extra_fields(self) -> dict[str, Any]:
- extra_fields = self.model_dump()
- extra_fields.update(self.extra_fields)
- return extra_fields
-
-
-ProblemStatementConfig = (
- TextProblemStatement | GithubIssue | EmptyProblemStatement | FileProblemStatement | CTFProblemStatement
-)
+ProblemStatementConfig = TextProblemStatement | GithubIssue | EmptyProblemStatement | FileProblemStatement
def problem_statement_from_simplified_input(
- *, input: str, type: Literal["text", "text_file", "github_issue", "ctf_json"]
+ *, input: str, type: Literal["text", "text_file", "github_issue"]
) -> ProblemStatementConfig:
"""Get a problem statement from an `input` string and a `type`.
@@ -190,8 +143,6 @@ def problem_statement_from_simplified_input(
return FileProblemStatement(path=Path(input))
elif type == "github_issue":
return GithubIssue(github_url=input)
- elif type == "ctf_json":
- return CTFProblemStatement(path=Path(input))
else:
msg = f"Unknown problem statement type: {type}"
raise ValueError(msg)
diff --git a/sweagent/environment/repo.py b/sweagent/environment/repo.py
index efcef7e2..0abf4438 100644
--- a/sweagent/environment/repo.py
+++ b/sweagent/environment/repo.py
@@ -6,7 +6,6 @@ from typing import Any, Literal, Protocol
from git import InvalidGitRepositoryError
from git import Repo as GitRepo
from pydantic import BaseModel, ConfigDict, Field
-from pydantic_core import from_json
from swerex.deployment.abstract import AbstractDeployment
from swerex.runtime.abstract import Command, UploadRequest
from typing_extensions import Self
@@ -182,53 +181,11 @@ class GithubRepoConfig(BaseModel):
return _get_git_reset_commands(self.base_commit)
-class CTFRepoConfig(BaseModel):
- path: Path
- challenge_json_filename: str = Field(default="challenge.json")
- """The JSON file containing the CTF challenge.
- This filename should be a relative path inside the repository parent path.
- """
-
- base_commit: str = Field(default="HEAD")
- """This field has no effect on CTF repo."""
-
- type: Literal["ctf"]
- """Discriminator for (de)serialization/CLI. Do not change."""
-
- model_config = ConfigDict(extra="forbid")
-
- @property
- def repo_name(self) -> str:
- """Set automatically based on the repository name. Cannot be set."""
- return Path(self.path).resolve().name.replace(" ", "-").replace("'", "")
-
- @property
- def files(self) -> list[str]:
- json_data = (self.path / self.challenge_json_filename).read_text()
- return from_json(json_data)["files"]
-
- def copy(self, deployment: AbstractDeployment):
- for file in self.files:
- asyncio.run(
- deployment.runtime.upload(
- UploadRequest(source_path=str(self.path / Path(file)), target_path=f"/{self.repo_name}/{file}")
- )
- )
- r = asyncio.run(deployment.runtime.execute(Command(command=f"chown -R root:root {self.repo_name}", shell=True)))
- if r.exit_code != 0:
- msg = f"Failed to change permissions on copied repository (exit code: {r.exit_code}, stdout: {r.stdout}, stderr: {r.stderr})"
- raise RuntimeError(msg)
-
- def get_reset_commands(self) -> list[str]:
- """Issued after the copy operation or when the environment is reset."""
- return ["git init"]
-
-
-RepoConfig = LocalRepoConfig | GithubRepoConfig | PreExistingRepoConfig | CTFRepoConfig
+RepoConfig = LocalRepoConfig | GithubRepoConfig | PreExistingRepoConfig
def repo_from_simplified_input(
- *, input: str, base_commit: str = "HEAD", type: Literal["local", "github", "preexisting", "auto", "ctf"] = "auto"
+ *, input: str, base_commit: str = "HEAD", type: Literal["local", "github", "preexisting", "auto"] = "auto"
) -> RepoConfig:
"""Get repo config from a simplified input.
@@ -248,7 +205,5 @@ def repo_from_simplified_input(
return GithubRepoConfig(github_url=input, base_commit=base_commit)
else:
return LocalRepoConfig(path=Path(input), base_commit=base_commit)
- if type == "ctf":
- return CTFRepoConfig(path=Path(input))
msg = f"Unknown repo type: {type}"
raise ValueError(msg)
diff --git a/sweagent/run/run_single.py b/sweagent/run/run_single.py
index bf528774..8ba4f77b 100644
--- a/sweagent/run/run_single.py
+++ b/sweagent/run/run_single.py
@@ -33,17 +33,15 @@ from pathlib import Path
from typing import Self
import yaml
-from pydantic import BaseModel, ConfigDict, Field, model_validator
+from pydantic import BaseModel, ConfigDict, Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from sweagent.agent.agents import AbstractAgent, AgentConfig, get_agent_from_config
from sweagent.agent.problem_statement import (
- CTFProblemStatement,
EmptyProblemStatement,
ProblemStatement,
ProblemStatementConfig,
)
-from sweagent.environment.repo import CTFRepoConfig
from sweagent.environment.swe_env import EnvironmentConfig, SWEEnv
from sweagent.run.common import AutoCorrectSuggestion as ACS
from sweagent.run.common import BasicCLI, ConfigHelper, save_predictions
@@ -118,12 +116,6 @@ class RunSingleConfig(BaseSettings, cli_implicit_flags=False):
ACS("repo.path", "env.repo.path"),
]
- @model_validator(mode="after")
- def check_repo_and_problem_statement_config(self) -> Self:
- if isinstance(self.problem_statement, CTFProblemStatement) != isinstance(self.env.repo, CTFRepoConfig):
- ValueError("CTF problem statement should be used only with CTF repo config!")
- return self
-
class RunSingle:
def __init__(
diff --git a/tests/conftest.py b/tests/conftest.py
index fabd72e8..b1314fa3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -50,14 +50,6 @@ def ctf_data_path(test_data_sources_path) -> Path:
return p
-def swe_agent_test_ctf_repo_and_ps(ctf_data_path) -> tuple[Path, Path]:
- p = ctf_data_path / "forensics" / "flash"
- ps = p / "challenge.json"
- assert p.is_dir()
- return p, ps
-
-
@pytest.fixture
def test_data_sources_path(test_data_path) -> Path:
p = test_data_path / "data_sources"
diff --git a/tests/test_run_single.py b/tests/test_run_single.py
index 185b08a2..59e8fff0 100644
--- a/tests/test_run_single.py
+++ b/tests/test_run_single.py
@@ -123,33 +123,3 @@ def test_run_ies_repo_ps_matrix(
for fmt in output_formats:
assert len(list(Path(tmpdir).rglob(f"*.{fmt}"))) == 1
print(fmt, list(Path(tmpdir).iterdir()))
-
-
-def test_run_single_ctf(tmpdir, swe_agent_test_ctf_repo_and_ps):
- # ies = instant empty submit
- output_formats = ["traj", "pred", "patch"]
- for fmt in output_formats:
- assert not list(Path(tmpdir).glob(f"*.{fmt}"))
- repo, ps = swe_agent_test_ctf_repo_and_ps
- ps_args = ["--problem_statement.path", str(ps), "--problem_statement.type", "ctf_json"]
- repo_args = ["--env.repo.path", str(repo), "--env.repo.type", "ctf"]
- args = [
- "--agent.model.name=instant_empty_submit",
- "--output_dir",
- str(tmpdir),
- *ps_args,
- *repo_args,
- "--config",
- str(CONFIG_DIR / "default_no_fcalls.yaml"),
- ]
- print(args)
- rs_config = BasicCLI(RunSingleConfig).get_config(args)
- print(rs_config)
- rs = RunSingle.from_config(rs_config) # type: ignore
- with tmpdir.as_cwd():
- # Test that we can run run.py also independently from repo dir
- rs.run()
- for fmt in output_formats:
- assert len(list(Path(tmpdir).rglob(f"*.{fmt}"))) == 1
- print(fmt, list(Path(tmpdir).iterdir()))