Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Feat: Most basic version of sweagent sh

Kilian Liereteeb8527e2a93
diff --git a/config/exotic/default_shell.yaml b/config/exotic/default_shell.yaml
new file mode 100644
index 00000000..e5484a56
--- /dev/null
+++ b/config/exotic/default_shell.yaml
@@ -0,0 +1,43 @@
+# For use with sweagent sh
+agent:
+ type: shell
+ templates:
+ system_template: |-
+ You are a helpful assistant that can interact with a computer to solve tasks.
+ instance_template: |-
+ <uploaded_files>
+ {{working_dir}}
+ </uploaded_files>
+ I've uploaded a python code repository in the directory {{working_dir}}. Consider the following PR description:
+
+ <pr_description>
+ {{problem_statement}}
+ </pr_description>
+
+ Can you help me implement the necessary changes to the repository so that the requirements specified in the <pr_description> are met?
+ I've already taken care of all changes to any of the test files described in the <pr_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+ Your task is to make the minimal changes to non-tests files in the {{working_dir}} directory to ensure the <pr_description> is satisfied.
+ Follow these steps to resolve the issue:
+ 1. As a first step, it might be a good idea to find and read code relevant to the <pr_description>
+ 2. Create a script to reproduce the error and execute it with `python <filename.py>` using the bash tool, to confirm the error
+ 3. Edit the sourcecode of the repo to resolve the issue
+ 4. Rerun your reproduce script and confirm that the error is fixed!
+ 5. Think about edgecases and make sure your fix handles them as well
+ Your thinking should be thorough and so it's fine if it's very long.
+ next_step_template: |-
+ OBSERVATION:
+ {{observation}}
+ next_step_no_output_template: |-
+ Your command ran successfully and did not produce any output.
+ tools:
+ bundles:
+ - path: tools/registry
+ - path: tools/edit_anthropic
+ registry_variables:
+ USE_FILEMAP: 'true'
+ enable_bash_tool: true
+ parse_function:
+ type: function_calling
+ history_processors:
+ - type: cache_control
+ last_n_messages: 2
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index 01549fe3..b2b0573f 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -161,6 +161,24 @@ class DefaultAgentConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
+class ShellAgentConfig(BaseModel):
+ name: str = "main"
+ templates: TemplateConfig = Field(default_factory=TemplateConfig)
+ tools: ToolConfig = Field(default_factory=ToolConfig)
+ history_processors: list[HistoryProcessor] = Field(default_factory=lambda: [DefaultHistoryProcessor()])
+ model: ModelConfig = Field(description="Model options.")
+
+ max_requeries: int = 3
+ """Maximum number of times to requery the model after an error, such as a
+ formatting error, a blocked action, or a bash syntax error.
+ """
+
+ type: Literal["shell"] = "shell"
+
+ # pydantic config
+ model_config = ConfigDict(extra="forbid")
+
+
class RetryAgentConfig(BaseModel):
name: str = "retry_main"
agent_configs: list[DefaultAgentConfig]
@@ -169,7 +187,7 @@ class RetryAgentConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
-AgentConfig = Annotated[DefaultAgentConfig | RetryAgentConfig, Field(union_mode="left_to_right")]
+AgentConfig = Annotated[DefaultAgentConfig | RetryAgentConfig | ShellAgentConfig, Field(union_mode="left_to_right")]
class _BlockedActionError(Exception):
@@ -220,6 +238,11 @@ def get_agent_from_config(config: AgentConfig) -> AbstractAgent:
return DefaultAgent.from_config(config)
elif config.type == "retry":
return RetryAgent.from_config(config)
+ elif config.type == "shell":
+ # Need to defer import to avoid circular dependency
+ from sweagent.agent.extra.shell_agent import ShellAgent
+
+ return ShellAgent.from_config(config)
else:
msg = f"Unknown agent type: {config.type}"
raise ValueError(msg)
diff --git a/sweagent/agent/extra/shell_agent.py b/sweagent/agent/extra/shell_agent.py
index 1a10e165..85935ccb 100644
--- a/sweagent/agent/extra/shell_agent.py
+++ b/sweagent/agent/extra/shell_agent.py
@@ -1,10 +1,12 @@
from pathlib import Path
+from typing import Self
-from sweagent.agent.agents import DefaultAgent
+from sweagent.agent.agents import DefaultAgent, ShellAgentConfig
from sweagent.agent.models import HumanModel, HumanModelConfig, get_model
from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig
from sweagent.environment.swe_env import SWEEnv
from sweagent.tools.parsing import ActionOnlyParser
+from sweagent.tools.tools import ToolHandler
from sweagent.types import AgentRunResult, StepOutput
@@ -13,6 +15,20 @@ class ShellAgent(DefaultAgent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ @classmethod
+ def from_config(cls, config: ShellAgentConfig) -> Self:
+ # To ensure that all models stay completely independent, we deepcopy the
+ # model config, because it lives on as a property in the model, tools, etc.
+ config = config.model_copy(deep=True)
+ model = get_model(config.model, config.tools)
+ return cls(
+ templates=config.templates,
+ tools=ToolHandler(config.tools),
+ history_processors=config.history_processors,
+ model=model,
+ max_requeries=config.max_requeries,
+ )
+
def human_step_in(self) -> None:
"""Replace the current model with a HumanModel instance.
This allows for human intervention during agent execution.
@@ -47,7 +63,6 @@ class ShellAgent(DefaultAgent):
problem_statement: ProblemStatement | ProblemStatementConfig,
*,
output_dir: Path = Path("."),
- interactive: bool = False,
) -> AgentRunResult:
"""Run the agent on a problem instance. This method contains the
main loop that repeatedly calls `self._step` until the problem is solved.
@@ -68,18 +83,16 @@ class ShellAgent(DefaultAgent):
step_output = self.step()
self.save_trajectory()
except KeyboardInterrupt:
- if interactive and not isinstance(self.model, HumanModel):
+ if not isinstance(self.model, HumanModel):
self.human_step_in()
continue
raise
except EOFError:
- if not interactive:
- raise
# Can only happen if we have a human model, so switch back
self.logger.info("Detected ^D - switching back to AI mode")
self.human_step_out()
continue
- if interactive and step_output.done and not isinstance(self.model, HumanModel):
+ if step_output.done and not isinstance(self.model, HumanModel):
# Human has to submit the solution
self.logger.info("Robot is done! Please submit the solution.")
self.human_step_in()
diff --git a/sweagent/run/common.py b/sweagent/run/common.py
index dd2d0c0c..37d7e1ed 100644
--- a/sweagent/run/common.py
+++ b/sweagent/run/common.py
@@ -185,7 +185,14 @@ def _parse_args_to_nested_dict(args):
# todo: Parameterize type hints
class BasicCLI:
- def __init__(self, config_type: type[BaseSettings], *, default_settings: bool = True, help_text: str | None = None):
+ def __init__(
+ self,
+ config_type: type[BaseSettings],
+ *,
+ default_settings: bool = True,
+ help_text: str | None = None,
+ default_config_file: Path = CONFIG_DIR / "default.yaml",
+ ):
"""This class implements a basic CLI for SWE-agent. It is based on pydantic-settings, i.e., takes
a `BaseSettings` object. In principle you could just initialize these via `pydantic-settings`'s `CliApp.run`,
however, we also want to add a `--config` option to load additional config files and some other things.
@@ -201,6 +208,7 @@ class BasicCLI:
self.default_settings = default_settings
self.logger = get_logger("swea-cli", emoji="🔧")
self.help_text = help_text
+ self.default_config_file = default_config_file
def maybe_show_auto_correct(self, args: list[str]):
auto_correct = []
@@ -289,7 +297,7 @@ class BasicCLI:
_loaded = yaml.safe_load(txt)
merge_nested_dicts(config_merged, _loaded)
elif self.default_settings and not cli_args.no_config_file:
- config_file = CONFIG_DIR / "default.yaml"
+ config_file = self.default_config_file
config_files.append(config_file)
msg = (
f"Loading default config from {config_file}, because no other "
diff --git a/sweagent/run/run.py b/sweagent/run/run.py
index 1806187d..e9952a9c 100644
--- a/sweagent/run/run.py
+++ b/sweagent/run/run.py
@@ -58,6 +58,8 @@ def get_cli():
"ru",
"quick-stats",
"qs",
+ "shell",
+ "sh",
],
nargs="?",
)
@@ -132,6 +134,10 @@ def main(args: list[str] | None = None):
from sweagent.run.quick_stats import run_from_cli as quick_stats_main
quick_stats_main(remaining_args)
+ elif command in ["shell", "sh"]:
+ from sweagent.run.run_shell import run_from_cli as run_shell_main
+
+ run_shell_main(remaining_args)
else:
msg = f"Unknown command: {command}"
raise ValueError(msg)
diff --git a/sweagent/run/run_shell.py b/sweagent/run/run_shell.py
index 85e46132..a50bf01e 100644
--- a/sweagent/run/run_shell.py
+++ b/sweagent/run/run_shell.py
@@ -11,15 +11,14 @@
"""
-import getpass
import sys
from pathlib import Path
from typing import Self
-import yaml
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
+from sweagent import CONFIG_DIR
from sweagent.agent.agents import AbstractAgent, AgentConfig, get_agent_from_config
from sweagent.agent.problem_statement import (
EmptyProblemStatement,
@@ -30,11 +29,12 @@ from sweagent.environment.swe_env import EnvironmentConfig, SWEEnv
from sweagent.run.common import AutoCorrectSuggestion as ACS
from sweagent.run.common import BasicCLI, ConfigHelper, save_predictions
from sweagent.run.hooks.abstract import CombinedRunHooks, RunHook
+from sweagent.run.run_single import _set_default_output_dir
from sweagent.utils.config import load_environment_variables
from sweagent.utils.log import add_file_handler, get_logger
-class RunSingleConfig(BaseSettings, cli_implicit_flags=False):
+class RunShellConfig(BaseSettings, cli_implicit_flags=False):
env: EnvironmentConfig = Field(default_factory=EnvironmentConfig, description="Environment options.")
agent: AgentConfig = Field(description="Agent options.")
problem_statement: ProblemStatementConfig = Field(
@@ -45,26 +45,13 @@ class RunSingleConfig(BaseSettings, cli_implicit_flags=False):
env_var_path: Path | None = None
"""Path to a .env file to load environment variables from."""
- interactive: bool = False
- """Whether to make the agent interactive (i.e., allow the human to jump in by pressing ^C)."""
-
# pydantic config
model_config = SettingsConfigDict(extra="forbid", env_prefix="SWE_AGENT_")
def set_default_output_dir(self) -> None:
# Needs to be called explicitly, because self._config_files will be setup
# post-init.
- if self.output_dir == Path("DEFAULT"):
- user_id = getpass.getuser()
- problem_id = self.problem_statement.id
- try:
- model_id = self.agent.model.id # type: ignore[attr-defined]
- except AttributeError:
- model_id = "unknown_model"
- config_file = getattr(self, "_config_files", ["no_config"])[0]
- if isinstance(config_file, Path):
- config_file = config_file.stem
- self.output_dir = Path.cwd() / "trajectories" / user_id / f"{config_file}__{model_id}___{problem_id}"
+ self.output_dir = _set_default_output_dir(self.output_dir, self.problem_statement, self.agent)
@classmethod
def _get_auto_correct(cls) -> list[ACS]:
@@ -87,7 +74,6 @@ class RunSingle:
*,
output_dir: Path = Path("."),
hooks: list[RunHook] | None = None,
- interactive: bool = False,
):
"""Note: When initializing this class, make sure to add the hooks that are required by your actions.
See `from_config` for an example.
@@ -109,14 +95,13 @@ class RunSingle:
self.problem_statement = problem_statement
for hook in hooks or []:
self.add_hook(hook)
- self.interactive = interactive
@property
def hooks(self) -> list[RunHook]:
return self._chooks.hooks
@classmethod
- def from_config(cls, config: RunSingleConfig) -> Self:
+ def from_config(cls, config: RunShellConfig) -> Self:
load_environment_variables(config.env_var_path)
config.set_default_output_dir()
config.output_dir.mkdir(parents=True, exist_ok=True)
@@ -127,7 +112,6 @@ class RunSingle:
agent=agent,
problem_statement=config.problem_statement,
output_dir=config.output_dir,
- interactive=config.interactive,
)
def add_hook(self, hook: RunHook) -> None:
@@ -142,8 +126,6 @@ class RunSingle:
self._chooks.on_instance_start(index=0, env=self.env, problem_statement=self.problem_statement)
output_dir = self.output_dir / self.problem_statement.id
output_dir.mkdir(parents=True, exist_ok=True)
- if self.agent.replay_config is not None: # type: ignore[attr-defined]
- (output_dir / "config.yaml").write_text(yaml.dump(self.agent.replay_config.model_dump_json(), indent=2)) # type: ignore[attr-defined]
result = self.agent.run(
problem_statement=self.problem_statement,
env=self.env,
@@ -156,7 +138,7 @@ class RunSingle:
self.env.close()
-def run_from_config(config: RunSingleConfig):
+def run_from_config(config: RunShellConfig):
RunSingle.from_config(config).run()
@@ -165,9 +147,13 @@ def run_from_cli(args: list[str] | None = None):
args = sys.argv[1:]
assert __doc__ is not None
help_text = ( # type: ignore
- __doc__ + "\n[cyan][bold]=== ALL THE OPTIONS ===[/bold][/cyan]\n\n" + ConfigHelper().get_help(RunSingleConfig)
+ __doc__ + "\n[cyan][bold]=== ALL THE OPTIONS ===[/bold][/cyan]\n\n" + ConfigHelper().get_help(RunShellConfig)
+ )
+ run_from_config(
+ BasicCLI( # type: ignore[reportUnknownReturnType]
+ RunShellConfig, help_text=help_text, default_config_file=CONFIG_DIR / "exotic" / "default_shell.yaml"
+ ).get_config(args)
)
- run_from_config(BasicCLI(RunSingleConfig, help_text=help_text).get_config(args)) # type: ignore
if __name__ == "__main__":
diff --git a/sweagent/run/run_single.py b/sweagent/run/run_single.py
index 8ba4f77b..2a478ee5 100644
--- a/sweagent/run/run_single.py
+++ b/sweagent/run/run_single.py
@@ -65,6 +65,21 @@ class RunSingleActionConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
+def _set_default_output_dir(output_dir: Path, problem_statement: ProblemStatement, agent: AgentConfig) -> Path:
+ if output_dir == Path("DEFAULT"):
+ user_id = getpass.getuser()
+ problem_id = problem_statement.id
+ try:
+ model_id = agent.model.id # type: ignore[attr-defined]
+ except AttributeError:
+ model_id = "unknown_model"
+ config_file = getattr(agent, "_config_files", ["no_config"])[0]
+ if isinstance(config_file, Path):
+ config_file = config_file.stem
+ return Path.cwd() / "trajectories" / user_id / f"{config_file}__{model_id}___{problem_id}"
+ return output_dir
+
+
class RunSingleConfig(BaseSettings, cli_implicit_flags=False):
env: EnvironmentConfig = Field(default_factory=EnvironmentConfig, description="Environment options.")
agent: AgentConfig = Field(description="Agent options.")
@@ -84,17 +99,7 @@ class RunSingleConfig(BaseSettings, cli_implicit_flags=False):
def set_default_output_dir(self) -> None:
# Needs to be called explicitly, because self._config_files will be setup
# post-init.
- if self.output_dir == Path("DEFAULT"):
- user_id = getpass.getuser()
- problem_id = self.problem_statement.id
- try:
- model_id = self.agent.model.id # type: ignore[attr-defined]
- except AttributeError:
- model_id = "unknown_model"
- config_file = getattr(self, "_config_files", ["no_config"])[0]
- if isinstance(config_file, Path):
- config_file = config_file.stem
- self.output_dir = Path.cwd() / "trajectories" / user_id / f"{config_file}__{model_id}___{problem_id}"
+ self.output_dir = _set_default_output_dir(self.output_dir, self.problem_statement, self.agent)
@classmethod
def _get_auto_correct(cls) -> list[ACS]: