SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
WIP
Kilian Lieretde239e89cef5
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex d8feff7b..fd451002 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -24,6 +24,7 @@ from sweagent.agent.models import (AbstractModel,HumanModel,HumanThoughtModel,+ InstanceStats,ModelConfig,get_model,)@@ -179,6 +180,10 @@ class AbstractAgent:def add_hook(self, hook: AbstractAgentHook) -> None: ...+ def get_trajectory_data(self) -> dict[str, Any]: ...++ def step(self) -> StepOutput: ...+def run(self, *args, **kwargs) -> AgentRunResult: ...@@ -188,53 +193,80 @@ class RetryAgent(AbstractAgent):self._hooks = []self._i_attempt = 0self.logger = get_logger("swea-agent", emoji="🤠")+ self._agent: DefaultAgent | None = None+ self._attempt_data: list[dict[str, Any]] = []+ self._total_instance_stats = InstanceStats()+ self._chook = CombinedAgentHook()+ self._traj_path: Path | None = None@classmethoddef from_config(cls, config: RetryAgentConfig) -> Self:return cls(config)def add_hook(self, hook: AbstractAgentHook) -> None:+ self._chook.add_hook(hook)self._hooks.append(hook)- def save_trajectory(self) -> None: ...-- def run(+ def setup(self, env: SWEEnv, problem_statement: ProblemStatement | ProblemStatementConfig, output_dir: Path = Path(".")- ) -> AgentRunResult:- output_dir.mkdir(parents=True, exist_ok=True)- self._chook.on_run_start()- self._rloop = get_retry_loop_from_config(self.config.retry_loop, instance=problem_statement)+ ) -> None:+ self.traj_path = output_dir / (self._problem_statement.id + ".traj")+ self._env = env+ self._problem_statement = problem_statement+ self._output_dir = output_dir+ self._rloop = get_retry_loop_from_config(self.config.retry_loop, problem_statement=problem_statement)++ def _setup_agent(self) -> AbstractAgent:+ # todo: Could select "best" agent config based on previous attempts if I run > number of set up configs+ self._agent = DefaultAgent.from_config(+ self.config.agent_configs[self._i_attempt % len(self.config.agent_configs)]+ )+ for hook in self._hooks:+ self._agent.add_hook(hook)+ sub_agent_output_dir = self._output_dir / f"attempt_{self._i_attempt}"+ self._agent.setup(env=self._env, problem_statement=self._problem_statement, output_dir=sub_agent_output_dir)+ return self._agent++ def _next_attempt(self) -> None:+ self._i_attempt += 1+ self._env.hard_reset()+ self._setup_agent()++ def step(self) -> StepOutput:+ assert self._agent is not None+ return self._agent.step()++ def _finalize_agent_run(self) -> None:+ assert self._agent is not None+ self._attempt_data.append(self._agent.get_trajectory_data())+ self._total_instance_stats += self._agent.model.stats++ def get_trajectory_data(self) -> dict[str, Any]:+ """Get all data that we save in .traj files."""assert self._rloop is not None- assert env is not None- while True:- agent = Agent.from_config(self.config.agent_configs[self._i_attempt])- # todo: change the output dir- attempt_output = agent.run(env, problem_statement, output_dir)- self._rloop.on_submit(ReviewSubmission(trajectory=attempt_output.trajectory, info=attempt_output.info))- attempt_output.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore- self.save_trajectory()- if not self._rloop.retry():- break- self._i_attempt += 1- env.hard_reset()- self._chook.on_run_done(trajectory=self.trajectory, info=self.info)+ best_attempt_idx = self._rloop.get_best()+ data = {+ "attempts": [self._attempt_data[i] for i in range(self._i_attempt + 1)],+ **self._attempt_data[best_attempt_idx],+ }+ data["info"]["best_attempt_idx"] = best_attempt_idx+ # Overwrite model stats with total stats+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()- self.logger.info("Trajectory saved to %s", self.traj_path)+ return data- # Here we want to return the "global" information (e.g., submission should- # be the best submission instead of the last one, etc.), so we get it from the traj file+ def save_trajectory(self) -> None:data = self.get_trajectory_data()- return AgentRunResult(info=data["info"], trajectory=data["trajectory"])+ assert self._traj_path is not None+ self._traj_path.write_text(json.dumps(data, indent=2))- return AgentRunResult(info=attempt_output.info, trajectory=attempt_output.trajectory)-- def _run_attempt(+ def run(self,env: SWEEnv,problem_statement: ProblemStatement | ProblemStatementConfig,output_dir: Path = Path("."),- ) -> StepOutput:+ ) -> AgentRunResult:"""Run the agent on a problem instance. This method contains themain loop that repeatedly calls `self._step` until the problem is solved.@@ -249,19 +281,21 @@ class RetryAgent(AbstractAgent):# Run action/observation loopself._chook.on_run_start()step_output = StepOutput()+ self._setup_agent()+ assert self._agent is not Nonewhile not step_output.done:step_output = self.step()self.save_trajectory()if step_output.done:if self._rloop is not None:- self._rloop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))+ self._rloop.on_submit(ReviewSubmission(trajectory=self._agent.trajectory, info=self._agent.info))self.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignoreself.save_trajectory()if self._rloop.retry():assert self._env is not None- self.setup_attempt(hard_reset=True)+ self._next_attempt()step_output.done = False- self._chook.on_run_done(trajectory=self.trajectory, info=self.info)+ self._chook.on_run_done(trajectory=self._agent.trajectory, info=self._agent.info)self.logger.info("Trajectory saved to %s", self.traj_path)@@ -271,7 +305,7 @@ class RetryAgent(AbstractAgent):return AgentRunResult(info=data["info"], trajectory=data["trajectory"])-class Agent(AbstractAgent):+class DefaultAgent(AbstractAgent):def __init__(self,*,diff --git a/sweagent/agent/hooks/abstract.py b/sweagent/agent/hooks/abstract.pyindex fa4b127c..57da5ba1 100644--- a/sweagent/agent/hooks/abstract.py+++ b/sweagent/agent/hooks/abstract.py@@ -4,11 +4,11 @@ from sweagent.types import AgentInfo, StepOutput, Trajectoryif TYPE_CHECKING:# avoid circular import- from sweagent.agent.agents import Agent+ from sweagent.agent.agents import DefaultAgentclass AbstractAgentHook:- def on_init(self, *, agent: "Agent"):+ def on_init(self, *, agent: "DefaultAgent"):"""Note: Depending on the internals of `Agent` should be done with care,it's best to use this as little as possible."""@@ -64,7 +64,7 @@ class CombinedAgentHook(AbstractAgentHook):def hooks(self) -> list[AbstractAgentHook]:return self._hooks- def on_init(self, *, agent: "Agent"):+ def on_init(self, *, agent: "DefaultAgent"):for hook in self.hooks:hook.on_init(agent=agent)diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex e6da8bf4..9442b4b8 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -142,8 +142,8 @@ class ScoreRetryLoopConfig(BaseModel):def __post_init__(self):self.validate()- def get_retry_loop(self, instance: ProblemStatement) -> AbstractRetryLoop:- return ScoreRetryLoop(self, instance)+ def get_retry_loop(self, problem_statement: ProblemStatement) -> AbstractRetryLoop:+ return ScoreRetryLoop(self, problem_statement)RetryLoopConfig = ScoreRetryLoopConfig@@ -273,10 +273,10 @@ class ScoreRetryLoop(AbstractRetryLoop):def __init__(self,loop_config: ScoreRetryLoopConfig,- instance: ProblemStatement,+ problem_statement: ProblemStatement,):self._model = get_model(loop_config.model, tools=ToolConfig())- self._instance = instance+ self._problem_statement = problem_statementself._reviewer: AbstractReviewer = loop_config.reviewer_config.get_reviewer(self._model)self._loop_config = loop_config# Note: These are "cumulative" submissions, i.e., they include all retries@@ -330,7 +330,7 @@ class ScoreRetryLoop(AbstractRetryLoop):raise AttemptCostLimitExceededError()def _review(self) -> float:- review = self._reviewer.review(self._instance, self._submissions[-1])+ review = self._reviewer.review(self._problem_statement, self._submissions[-1])self._reviews.append(review)exit_status = self._submissions[-1].info.get("exit_status", "")if exit_status and "exit_cost" in exit_status.lower():@@ -385,7 +385,9 @@ class ScoreRetryLoop(AbstractRetryLoop):return best_indices[0]-def get_retry_loop_from_config(config: RetryLoopConfig | None, instance: ProblemStatement) -> AbstractRetryLoop | None:+def get_retry_loop_from_config(+ config: RetryLoopConfig | None, problem_statement: ProblemStatement+) -> AbstractRetryLoop | None:if config is None:return None- return config.get_retry_loop(instance=instance)+ return config.get_retry_loop(problem_statement=problem_statement)diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.pyindex 54514b73..e59e97f9 100644--- a/sweagent/run/run_batch.py+++ b/sweagent/run/run_batch.py@@ -47,7 +47,7 @@ from pydantic_settings import BaseSettingsfrom rich.live import Livefrom swerex.deployment.hooks.status import SetStatusDeploymentHook-from sweagent.agent.agents import Agent, AgentConfig+from sweagent.agent.agents import AgentConfig, DefaultAgentfrom sweagent.agent.hooks.status import SetStatusAgentHookfrom sweagent.environment.hooks.status import SetStatusEnvironmentHookfrom sweagent.environment.swe_env import SWEEnv@@ -308,7 +308,7 @@ class RunBatch:output_dir = Path(self.output_dir) / instance.problem_statement.idoutput_dir.mkdir(parents=True, exist_ok=True)self.agent_config.name = f"{instance.problem_statement.id}"- agent = Agent.from_config(self.agent_config)+ agent = DefaultAgent.from_config(self.agent_config)single_run_replay_config = RunSingleConfig(agent=self.agent_config,problem_statement=instance.problem_statement,diff --git a/sweagent/run/run_replay.py b/sweagent/run/run_replay.pyindex f3136644..48fa8e89 100644--- a/sweagent/run/run_replay.py+++ b/sweagent/run/run_replay.py@@ -34,7 +34,7 @@ from swerex.deployment.abstract import AbstractDeploymentfrom swerex.deployment.config import DeploymentConfig, get_deploymentfrom typing_extensions import Self-from sweagent.agent.agents import Agent+from sweagent.agent.agents import DefaultAgentfrom sweagent.agent.models import ReplayModelConfigfrom sweagent.environment.swe_env import SWEEnvfrom sweagent.run.common import BasicCLI, ConfigHelper@@ -174,8 +174,8 @@ class RunReplay:post_startup_commands=[],)- def _get_agent(self) -> Agent:- agent = Agent.from_config(self.config.agent)+ def _get_agent(self) -> DefaultAgent:+ agent = DefaultAgent.from_config(self.config.agent)agent._catch_errors = self._catch_errorsagent._always_require_zero_exit_code = self._require_zero_exit_codereturn agentdiff --git a/sweagent/run/run_single.py b/sweagent/run/run_single.pyindex 30ac3119..2548c433 100644--- a/sweagent/run/run_single.py+++ b/sweagent/run/run_single.py@@ -36,7 +36,7 @@ import yamlfrom pydantic import BaseModel, ConfigDict, Fieldfrom pydantic_settings import BaseSettings-from sweagent.agent.agents import Agent, AgentConfig+from sweagent.agent.agents import AgentConfig, DefaultAgentfrom sweagent.agent.problem_statement import (EmptyProblemStatement,ProblemStatement,@@ -114,7 +114,7 @@ class RunSingle:def __init__(self,env: SWEEnv,- agent: Agent,+ agent: DefaultAgent,problem_statement: ProblemStatement | ProblemStatementConfig,*,output_dir: Path = Path("."),@@ -154,7 +154,7 @@ class RunSingle:load_environment_variables(config.env_var_path)config.set_default_output_dir()config.output_dir.mkdir(parents=True, exist_ok=True)- agent = Agent.from_config(config.agent)+ agent = DefaultAgent.from_config(config.agent)agent.replay_config = configself = cls(env=SWEEnv.from_config(config.env),diff --git a/tests/test_agent.py b/tests/test_agent.pyindex a07e9070..84ef1a87 100644--- a/tests/test_agent.py+++ b/tests/test_agent.py@@ -5,7 +5,7 @@ from swerex.runtime.abstract import Action, BashObservation, Observationfrom swerex.runtime.dummy import DummyRuntimefrom sweagent import CONFIG_DIR-from sweagent.agent.agents import Agent, AgentConfig+from sweagent.agent.agents import DefaultAgent, DefaultAgentConfigfrom sweagent.agent.models import InstantEmptySubmitModelConfig, PredeterminedTestModelfrom sweagent.agent.problem_statement import EmptyProblemStatement, TextProblemStatementfrom sweagent.environment.swe_env import SWEEnv@@ -19,7 +19,7 @@ def test_dummy_env(dummy_env):@pytest.fixturedef identity_agent_config():- return AgentConfig(+ return DefaultAgentConfig(model=InstantEmptySubmitModelConfig(),tools=ToolConfig(parse_function=Identity(),@@ -29,7 +29,7 @@ def identity_agent_config():@pytest.fixturedef thought_action_agent_config():- return AgentConfig(+ return DefaultAgentConfig(model=InstantEmptySubmitModelConfig(),tools=ToolConfig(parse_function=ThoughtActionParser(),@@ -39,7 +39,7 @@ def thought_action_agent_config():@pytest.fixturedef function_calling_agent_config():- return AgentConfig(+ return DefaultAgentConfig(model=InstantEmptySubmitModelConfig(),tools=ToolConfig(parse_function=FunctionCallingParser(),@@ -52,32 +52,32 @@ def default_agent_config():config = yaml.safe_load((CONFIG_DIR / "default_no_fcalls.yaml").read_text())config["agent"]["model"] = {"name": "instant_empty_submit"}print(yaml.dump(config))- return AgentConfig.model_validate(config["agent"])+ return DefaultAgentConfig.model_validate(config["agent"])@pytest.fixture-def default_agent(default_agent_config: AgentConfig) -> Agent:- a = Agent.from_config(default_agent_config)+def default_agent(default_agent_config: DefaultAgentConfig) -> DefaultAgent:+ a = DefaultAgent.from_config(default_agent_config)a.tools.mock_state = {"open_file": "asdf123", "working_dir": "/root"}return a@pytest.fixture-def test_agent(identity_agent_config: AgentConfig) -> Agent:- return Agent.from_config(identity_agent_config)+def test_agent(identity_agent_config: DefaultAgentConfig) -> DefaultAgent:+ return DefaultAgent.from_config(identity_agent_config)@pytest.fixture-def thought_action_agent(thought_action_agent_config: AgentConfig) -> Agent:- return Agent.from_config(thought_action_agent_config)+def thought_action_agent(thought_action_agent_config: DefaultAgentConfig) -> DefaultAgent:+ return DefaultAgent.from_config(thought_action_agent_config)@pytest.fixture-def function_calling_agent(function_calling_agent_config: AgentConfig) -> Agent:- return Agent.from_config(function_calling_agent_config)+def function_calling_agent(function_calling_agent_config: DefaultAgentConfig) -> DefaultAgent:+ return DefaultAgent.from_config(function_calling_agent_config)-def test_exit_cost(dummy_env: SWEEnv, test_agent: Agent, tmp_path):+def test_exit_cost(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):test_agent.model = PredeterminedTestModel(["raise_cost"]) # type: ignorer = test_agent.run(problem_statement=EmptyProblemStatement(),@@ -87,7 +87,7 @@ def test_exit_cost(dummy_env: SWEEnv, test_agent: Agent, tmp_path):assert r.info["exit_status"] == "exit_cost" # type: ignore-def test_exit_context(dummy_env: SWEEnv, test_agent: Agent, tmp_path):+def test_exit_context(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):test_agent.model = PredeterminedTestModel(["raise_context"]) # type: ignorer = test_agent.run(problem_statement=EmptyProblemStatement(),@@ -97,7 +97,7 @@ def test_exit_context(dummy_env: SWEEnv, test_agent: Agent, tmp_path):assert r.info["exit_status"] == "exit_context" # type: ignore-def test_exit_model_error(dummy_env: SWEEnv, test_agent: Agent, tmp_path):+def test_exit_model_error(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):test_agent.model = PredeterminedTestModel(["raise_runtime"]) # type: ignorer = test_agent.run(problem_statement=EmptyProblemStatement(),@@ -107,7 +107,7 @@ def test_exit_model_error(dummy_env: SWEEnv, test_agent: Agent, tmp_path):assert r.info["exit_status"] == "exit_environment_error" # type: ignore-def test_exit_format(dummy_env: SWEEnv, thought_action_agent: Agent, tmp_path):+def test_exit_format(dummy_env: SWEEnv, thought_action_agent: DefaultAgent, tmp_path):thought_action_agent.model = PredeterminedTestModel(["a", "b", "c", "d"]) # type: ignorer = thought_action_agent.run(problem_statement=EmptyProblemStatement(),@@ -117,7 +117,7 @@ def test_exit_format(dummy_env: SWEEnv, thought_action_agent: Agent, tmp_path):assert r.info["exit_status"] == "exit_format" # type: ignore-def test_exit_blocklist(dummy_env: SWEEnv, test_agent: Agent, tmp_path):+def test_exit_blocklist(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):test_agent.model = PredeterminedTestModel(["vim", "python", "su", "nano"]) # type: ignorer = test_agent.run(problem_statement=EmptyProblemStatement(),@@ -134,7 +134,7 @@ class RuntimeRaisesFirst(DummyRuntime):return await super().run_in_session(action)-def test_early_exit(dummy_env: SWEEnv, test_agent: Agent, tmp_path):+def test_early_exit(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):test_agent.model = PredeterminedTestModel(["raise"]) # type: ignoretest_agent._catch_errors = Truedummy_env.deployment.runtime = RuntimeRaisesFirst() # type: ignore@@ -146,7 +146,7 @@ def test_early_exit(dummy_env: SWEEnv, test_agent: Agent, tmp_path):assert r.info["exit_status"] == "exit_environment_error" # type: ignore-def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: Agent, tmp_path):+def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):a = default_agenta.model = PredeterminedTestModel(["asdf", "```\nls\n```", "```\necho 'asdf'\n```", "raise_cost"]) # type: ignorea.setup(dummy_env, TextProblemStatement(text="asdf123"))@@ -188,7 +188,7 @@ def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: Age# todo: fixme; Needs real environment or mocking of read_file@pytest.mark.xfail-def test_run_autosubmit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):+def test_run_autosubmit(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):a = default_agenta.model = PredeterminedTestModel(["raise_cost"]) # type: ignorea.setup(dummy_env, EmptyProblemStatement())@@ -208,7 +208,7 @@ def test_run_autosubmit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):assert "cost limit" in r.thought-def test_show_no_output_template(dummy_env: SWEEnv, default_agent: Agent, tmp_path):+def test_show_no_output_template(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):a = default_agenta.templates.next_step_no_output_template = "no output template"a.setup(dummy_env, EmptyProblemStatement())@@ -221,7 +221,7 @@ def test_show_no_output_template(dummy_env: SWEEnv, default_agent: Agent, tmp_pa# todo: fixme; Needs real environment or mocking of read_file@pytest.mark.xfail-def test_successful_submission(dummy_env: SWEEnv, default_agent: Agent, tmp_path):+def test_successful_submission(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):a = default_agenta.model = PredeterminedTestModel(["```\nsubmit\n```"]) # type: ignorea.setup(dummy_env, EmptyProblemStatement())@@ -233,7 +233,7 @@ def test_successful_submission(dummy_env: SWEEnv, default_agent: Agent, tmp_pathassert a.trajectory[-1]["observation"] == "test"-def test_human_exit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):+def test_human_exit(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):a = default_agenta.model = PredeterminedTestModel(["```\nexit\n```"]) # type: ignorea.setup(dummy_env, EmptyProblemStatement())@@ -243,7 +243,7 @@ def test_human_exit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):assert r.action.strip() == "exit"-def test_function_calling(dummy_env: SWEEnv, function_calling_agent: Agent, tmp_path):+def test_function_calling(dummy_env: SWEEnv, function_calling_agent: DefaultAgent, tmp_path):a = function_calling_agent# Simulate a valid function call response from the modelvalid_response = {