Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

WIP

Kilian Lieretde239e89cef5
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index d8feff7b..fd451002 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -24,6 +24,7 @@ from sweagent.agent.models import (
AbstractModel,
HumanModel,
HumanThoughtModel,
+ InstanceStats,
ModelConfig,
get_model,
)
@@ -179,6 +180,10 @@ class AbstractAgent:
def add_hook(self, hook: AbstractAgentHook) -> None: ...
+ def get_trajectory_data(self) -> dict[str, Any]: ...
+
+ def step(self) -> StepOutput: ...
+
def run(self, *args, **kwargs) -> AgentRunResult: ...
@@ -188,53 +193,80 @@ class RetryAgent(AbstractAgent):
self._hooks = []
self._i_attempt = 0
self.logger = get_logger("swea-agent", emoji="🤠")
+ self._agent: DefaultAgent | None = None
+ self._attempt_data: list[dict[str, Any]] = []
+ self._total_instance_stats = InstanceStats()
+ self._chook = CombinedAgentHook()
+ self._traj_path: Path | None = None
@classmethod
def from_config(cls, config: RetryAgentConfig) -> Self:
return cls(config)
def add_hook(self, hook: AbstractAgentHook) -> None:
+ self._chook.add_hook(hook)
self._hooks.append(hook)
- def save_trajectory(self) -> None: ...
-
- def run(
+ def setup(
self, env: SWEEnv, problem_statement: ProblemStatement | ProblemStatementConfig, output_dir: Path = Path(".")
- ) -> AgentRunResult:
- output_dir.mkdir(parents=True, exist_ok=True)
- self._chook.on_run_start()
- self._rloop = get_retry_loop_from_config(self.config.retry_loop, instance=problem_statement)
+ ) -> None:
+ self.traj_path = output_dir / (self._problem_statement.id + ".traj")
+ self._env = env
+ self._problem_statement = problem_statement
+ self._output_dir = output_dir
+ self._rloop = get_retry_loop_from_config(self.config.retry_loop, problem_statement=problem_statement)
+
+ def _setup_agent(self) -> AbstractAgent:
+ # todo: Could select "best" agent config based on previous attempts if I run > number of set up configs
+ self._agent = DefaultAgent.from_config(
+ self.config.agent_configs[self._i_attempt % len(self.config.agent_configs)]
+ )
+ for hook in self._hooks:
+ self._agent.add_hook(hook)
+ sub_agent_output_dir = self._output_dir / f"attempt_{self._i_attempt}"
+ self._agent.setup(env=self._env, problem_statement=self._problem_statement, output_dir=sub_agent_output_dir)
+ return self._agent
+
+ def _next_attempt(self) -> None:
+ self._i_attempt += 1
+ self._env.hard_reset()
+ self._setup_agent()
+
+ def step(self) -> StepOutput:
+ assert self._agent is not None
+ return self._agent.step()
+
+ def _finalize_agent_run(self) -> None:
+ assert self._agent is not None
+ self._attempt_data.append(self._agent.get_trajectory_data())
+ self._total_instance_stats += self._agent.model.stats
+
+ def get_trajectory_data(self) -> dict[str, Any]:
+ """Get all data that we save in .traj files."""
assert self._rloop is not None
- assert env is not None
- while True:
- agent = Agent.from_config(self.config.agent_configs[self._i_attempt])
- # todo: change the output dir
- attempt_output = agent.run(env, problem_statement, output_dir)
- self._rloop.on_submit(ReviewSubmission(trajectory=attempt_output.trajectory, info=attempt_output.info))
- attempt_output.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore
- self.save_trajectory()
- if not self._rloop.retry():
- break
- self._i_attempt += 1
- env.hard_reset()
- self._chook.on_run_done(trajectory=self.trajectory, info=self.info)
+ best_attempt_idx = self._rloop.get_best()
+ data = {
+ "attempts": [self._attempt_data[i] for i in range(self._i_attempt + 1)],
+ **self._attempt_data[best_attempt_idx],
+ }
+ data["info"]["best_attempt_idx"] = best_attempt_idx
+ # Overwrite model stats with total stats
+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()
- self.logger.info("Trajectory saved to %s", self.traj_path)
+ return data
- # Here we want to return the "global" information (e.g., submission should
- # be the best submission instead of the last one, etc.), so we get it from the traj file
+ def save_trajectory(self) -> None:
data = self.get_trajectory_data()
- return AgentRunResult(info=data["info"], trajectory=data["trajectory"])
+ assert self._traj_path is not None
+ self._traj_path.write_text(json.dumps(data, indent=2))
- return AgentRunResult(info=attempt_output.info, trajectory=attempt_output.trajectory)
-
- def _run_attempt(
+ def run(
self,
env: SWEEnv,
problem_statement: ProblemStatement | ProblemStatementConfig,
output_dir: Path = Path("."),
- ) -> StepOutput:
+ ) -> AgentRunResult:
"""Run the agent on a problem instance. This method contains the
main loop that repeatedly calls `self._step` until the problem is solved.
@@ -249,19 +281,21 @@ class RetryAgent(AbstractAgent):
# Run action/observation loop
self._chook.on_run_start()
step_output = StepOutput()
+ self._setup_agent()
+ assert self._agent is not None
while not step_output.done:
step_output = self.step()
self.save_trajectory()
if step_output.done:
if self._rloop is not None:
- self._rloop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))
+ self._rloop.on_submit(ReviewSubmission(trajectory=self._agent.trajectory, info=self._agent.info))
self.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore
self.save_trajectory()
if self._rloop.retry():
assert self._env is not None
- self.setup_attempt(hard_reset=True)
+ self._next_attempt()
step_output.done = False
- self._chook.on_run_done(trajectory=self.trajectory, info=self.info)
+ self._chook.on_run_done(trajectory=self._agent.trajectory, info=self._agent.info)
self.logger.info("Trajectory saved to %s", self.traj_path)
@@ -271,7 +305,7 @@ class RetryAgent(AbstractAgent):
return AgentRunResult(info=data["info"], trajectory=data["trajectory"])
-class Agent(AbstractAgent):
+class DefaultAgent(AbstractAgent):
def __init__(
self,
*,
diff --git a/sweagent/agent/hooks/abstract.py b/sweagent/agent/hooks/abstract.py
index fa4b127c..57da5ba1 100644
--- a/sweagent/agent/hooks/abstract.py
+++ b/sweagent/agent/hooks/abstract.py
@@ -4,11 +4,11 @@ from sweagent.types import AgentInfo, StepOutput, Trajectory
if TYPE_CHECKING:
# avoid circular import
- from sweagent.agent.agents import Agent
+ from sweagent.agent.agents import DefaultAgent
class AbstractAgentHook:
- def on_init(self, *, agent: "Agent"):
+ def on_init(self, *, agent: "DefaultAgent"):
"""Note: Depending on the internals of `Agent` should be done with care,
it's best to use this as little as possible.
"""
@@ -64,7 +64,7 @@ class CombinedAgentHook(AbstractAgentHook):
def hooks(self) -> list[AbstractAgentHook]:
return self._hooks
- def on_init(self, *, agent: "Agent"):
+ def on_init(self, *, agent: "DefaultAgent"):
for hook in self.hooks:
hook.on_init(agent=agent)
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index e6da8bf4..9442b4b8 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -142,8 +142,8 @@ class ScoreRetryLoopConfig(BaseModel):
def __post_init__(self):
self.validate()
- def get_retry_loop(self, instance: ProblemStatement) -> AbstractRetryLoop:
- return ScoreRetryLoop(self, instance)
+ def get_retry_loop(self, problem_statement: ProblemStatement) -> AbstractRetryLoop:
+ return ScoreRetryLoop(self, problem_statement)
RetryLoopConfig = ScoreRetryLoopConfig
@@ -273,10 +273,10 @@ class ScoreRetryLoop(AbstractRetryLoop):
def __init__(
self,
loop_config: ScoreRetryLoopConfig,
- instance: ProblemStatement,
+ problem_statement: ProblemStatement,
):
self._model = get_model(loop_config.model, tools=ToolConfig())
- self._instance = instance
+ self._problem_statement = problem_statement
self._reviewer: AbstractReviewer = loop_config.reviewer_config.get_reviewer(self._model)
self._loop_config = loop_config
# Note: These are "cumulative" submissions, i.e., they include all retries
@@ -330,7 +330,7 @@ class ScoreRetryLoop(AbstractRetryLoop):
raise AttemptCostLimitExceededError()
def _review(self) -> float:
- review = self._reviewer.review(self._instance, self._submissions[-1])
+ review = self._reviewer.review(self._problem_statement, self._submissions[-1])
self._reviews.append(review)
exit_status = self._submissions[-1].info.get("exit_status", "")
if exit_status and "exit_cost" in exit_status.lower():
@@ -385,7 +385,9 @@ class ScoreRetryLoop(AbstractRetryLoop):
return best_indices[0]
-def get_retry_loop_from_config(config: RetryLoopConfig | None, instance: ProblemStatement) -> AbstractRetryLoop | None:
+def get_retry_loop_from_config(
+ config: RetryLoopConfig | None, problem_statement: ProblemStatement
+) -> AbstractRetryLoop | None:
if config is None:
return None
- return config.get_retry_loop(instance=instance)
+ return config.get_retry_loop(problem_statement=problem_statement)
diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.py
index 54514b73..e59e97f9 100644
--- a/sweagent/run/run_batch.py
+++ b/sweagent/run/run_batch.py
@@ -47,7 +47,7 @@ from pydantic_settings import BaseSettings
from rich.live import Live
from swerex.deployment.hooks.status import SetStatusDeploymentHook
-from sweagent.agent.agents import Agent, AgentConfig
+from sweagent.agent.agents import AgentConfig, DefaultAgent
from sweagent.agent.hooks.status import SetStatusAgentHook
from sweagent.environment.hooks.status import SetStatusEnvironmentHook
from sweagent.environment.swe_env import SWEEnv
@@ -308,7 +308,7 @@ class RunBatch:
output_dir = Path(self.output_dir) / instance.problem_statement.id
output_dir.mkdir(parents=True, exist_ok=True)
self.agent_config.name = f"{instance.problem_statement.id}"
- agent = Agent.from_config(self.agent_config)
+ agent = DefaultAgent.from_config(self.agent_config)
single_run_replay_config = RunSingleConfig(
agent=self.agent_config,
problem_statement=instance.problem_statement,
diff --git a/sweagent/run/run_replay.py b/sweagent/run/run_replay.py
index f3136644..48fa8e89 100644
--- a/sweagent/run/run_replay.py
+++ b/sweagent/run/run_replay.py
@@ -34,7 +34,7 @@ from swerex.deployment.abstract import AbstractDeployment
from swerex.deployment.config import DeploymentConfig, get_deployment
from typing_extensions import Self
-from sweagent.agent.agents import Agent
+from sweagent.agent.agents import DefaultAgent
from sweagent.agent.models import ReplayModelConfig
from sweagent.environment.swe_env import SWEEnv
from sweagent.run.common import BasicCLI, ConfigHelper
@@ -174,8 +174,8 @@ class RunReplay:
post_startup_commands=[],
)
- def _get_agent(self) -> Agent:
- agent = Agent.from_config(self.config.agent)
+ def _get_agent(self) -> DefaultAgent:
+ agent = DefaultAgent.from_config(self.config.agent)
agent._catch_errors = self._catch_errors
agent._always_require_zero_exit_code = self._require_zero_exit_code
return agent
diff --git a/sweagent/run/run_single.py b/sweagent/run/run_single.py
index 30ac3119..2548c433 100644
--- a/sweagent/run/run_single.py
+++ b/sweagent/run/run_single.py
@@ -36,7 +36,7 @@ import yaml
from pydantic import BaseModel, ConfigDict, Field
from pydantic_settings import BaseSettings
-from sweagent.agent.agents import Agent, AgentConfig
+from sweagent.agent.agents import AgentConfig, DefaultAgent
from sweagent.agent.problem_statement import (
EmptyProblemStatement,
ProblemStatement,
@@ -114,7 +114,7 @@ class RunSingle:
def __init__(
self,
env: SWEEnv,
- agent: Agent,
+ agent: DefaultAgent,
problem_statement: ProblemStatement | ProblemStatementConfig,
*,
output_dir: Path = Path("."),
@@ -154,7 +154,7 @@ class RunSingle:
load_environment_variables(config.env_var_path)
config.set_default_output_dir()
config.output_dir.mkdir(parents=True, exist_ok=True)
- agent = Agent.from_config(config.agent)
+ agent = DefaultAgent.from_config(config.agent)
agent.replay_config = config
self = cls(
env=SWEEnv.from_config(config.env),
diff --git a/tests/test_agent.py b/tests/test_agent.py
index a07e9070..84ef1a87 100644
--- a/tests/test_agent.py
+++ b/tests/test_agent.py
@@ -5,7 +5,7 @@ from swerex.runtime.abstract import Action, BashObservation, Observation
from swerex.runtime.dummy import DummyRuntime
from sweagent import CONFIG_DIR
-from sweagent.agent.agents import Agent, AgentConfig
+from sweagent.agent.agents import DefaultAgent, DefaultAgentConfig
from sweagent.agent.models import InstantEmptySubmitModelConfig, PredeterminedTestModel
from sweagent.agent.problem_statement import EmptyProblemStatement, TextProblemStatement
from sweagent.environment.swe_env import SWEEnv
@@ -19,7 +19,7 @@ def test_dummy_env(dummy_env):
@pytest.fixture
def identity_agent_config():
- return AgentConfig(
+ return DefaultAgentConfig(
model=InstantEmptySubmitModelConfig(),
tools=ToolConfig(
parse_function=Identity(),
@@ -29,7 +29,7 @@ def identity_agent_config():
@pytest.fixture
def thought_action_agent_config():
- return AgentConfig(
+ return DefaultAgentConfig(
model=InstantEmptySubmitModelConfig(),
tools=ToolConfig(
parse_function=ThoughtActionParser(),
@@ -39,7 +39,7 @@ def thought_action_agent_config():
@pytest.fixture
def function_calling_agent_config():
- return AgentConfig(
+ return DefaultAgentConfig(
model=InstantEmptySubmitModelConfig(),
tools=ToolConfig(
parse_function=FunctionCallingParser(),
@@ -52,32 +52,32 @@ def default_agent_config():
config = yaml.safe_load((CONFIG_DIR / "default_no_fcalls.yaml").read_text())
config["agent"]["model"] = {"name": "instant_empty_submit"}
print(yaml.dump(config))
- return AgentConfig.model_validate(config["agent"])
+ return DefaultAgentConfig.model_validate(config["agent"])
@pytest.fixture
-def default_agent(default_agent_config: AgentConfig) -> Agent:
- a = Agent.from_config(default_agent_config)
+def default_agent(default_agent_config: DefaultAgentConfig) -> DefaultAgent:
+ a = DefaultAgent.from_config(default_agent_config)
a.tools.mock_state = {"open_file": "asdf123", "working_dir": "/root"}
return a
@pytest.fixture
-def test_agent(identity_agent_config: AgentConfig) -> Agent:
- return Agent.from_config(identity_agent_config)
+def test_agent(identity_agent_config: DefaultAgentConfig) -> DefaultAgent:
+ return DefaultAgent.from_config(identity_agent_config)
@pytest.fixture
-def thought_action_agent(thought_action_agent_config: AgentConfig) -> Agent:
- return Agent.from_config(thought_action_agent_config)
+def thought_action_agent(thought_action_agent_config: DefaultAgentConfig) -> DefaultAgent:
+ return DefaultAgent.from_config(thought_action_agent_config)
@pytest.fixture
-def function_calling_agent(function_calling_agent_config: AgentConfig) -> Agent:
- return Agent.from_config(function_calling_agent_config)
+def function_calling_agent(function_calling_agent_config: DefaultAgentConfig) -> DefaultAgent:
+ return DefaultAgent.from_config(function_calling_agent_config)
-def test_exit_cost(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
+def test_exit_cost(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):
test_agent.model = PredeterminedTestModel(["raise_cost"]) # type: ignore
r = test_agent.run(
problem_statement=EmptyProblemStatement(),
@@ -87,7 +87,7 @@ def test_exit_cost(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
assert r.info["exit_status"] == "exit_cost" # type: ignore
-def test_exit_context(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
+def test_exit_context(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):
test_agent.model = PredeterminedTestModel(["raise_context"]) # type: ignore
r = test_agent.run(
problem_statement=EmptyProblemStatement(),
@@ -97,7 +97,7 @@ def test_exit_context(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
assert r.info["exit_status"] == "exit_context" # type: ignore
-def test_exit_model_error(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
+def test_exit_model_error(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):
test_agent.model = PredeterminedTestModel(["raise_runtime"]) # type: ignore
r = test_agent.run(
problem_statement=EmptyProblemStatement(),
@@ -107,7 +107,7 @@ def test_exit_model_error(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
assert r.info["exit_status"] == "exit_environment_error" # type: ignore
-def test_exit_format(dummy_env: SWEEnv, thought_action_agent: Agent, tmp_path):
+def test_exit_format(dummy_env: SWEEnv, thought_action_agent: DefaultAgent, tmp_path):
thought_action_agent.model = PredeterminedTestModel(["a", "b", "c", "d"]) # type: ignore
r = thought_action_agent.run(
problem_statement=EmptyProblemStatement(),
@@ -117,7 +117,7 @@ def test_exit_format(dummy_env: SWEEnv, thought_action_agent: Agent, tmp_path):
assert r.info["exit_status"] == "exit_format" # type: ignore
-def test_exit_blocklist(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
+def test_exit_blocklist(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):
test_agent.model = PredeterminedTestModel(["vim", "python", "su", "nano"]) # type: ignore
r = test_agent.run(
problem_statement=EmptyProblemStatement(),
@@ -134,7 +134,7 @@ class RuntimeRaisesFirst(DummyRuntime):
return await super().run_in_session(action)
-def test_early_exit(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
+def test_early_exit(dummy_env: SWEEnv, test_agent: DefaultAgent, tmp_path):
test_agent.model = PredeterminedTestModel(["raise"]) # type: ignore
test_agent._catch_errors = True
dummy_env.deployment.runtime = RuntimeRaisesFirst() # type: ignore
@@ -146,7 +146,7 @@ def test_early_exit(dummy_env: SWEEnv, test_agent: Agent, tmp_path):
assert r.info["exit_status"] == "exit_environment_error" # type: ignore
-def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
+def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["asdf", "```\nls\n```", "```\necho 'asdf'\n```", "raise_cost"]) # type: ignore
a.setup(dummy_env, TextProblemStatement(text="asdf123"))
@@ -188,7 +188,7 @@ def test_run_step_by_step_checking_history(dummy_env: SWEEnv, default_agent: Age
# todo: fixme; Needs real environment or mocking of read_file
@pytest.mark.xfail
-def test_run_autosubmit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
+def test_run_autosubmit(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["raise_cost"]) # type: ignore
a.setup(dummy_env, EmptyProblemStatement())
@@ -208,7 +208,7 @@ def test_run_autosubmit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
assert "cost limit" in r.thought
-def test_show_no_output_template(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
+def test_show_no_output_template(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):
a = default_agent
a.templates.next_step_no_output_template = "no output template"
a.setup(dummy_env, EmptyProblemStatement())
@@ -221,7 +221,7 @@ def test_show_no_output_template(dummy_env: SWEEnv, default_agent: Agent, tmp_pa
# todo: fixme; Needs real environment or mocking of read_file
@pytest.mark.xfail
-def test_successful_submission(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
+def test_successful_submission(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["```\nsubmit\n```"]) # type: ignore
a.setup(dummy_env, EmptyProblemStatement())
@@ -233,7 +233,7 @@ def test_successful_submission(dummy_env: SWEEnv, default_agent: Agent, tmp_path
assert a.trajectory[-1]["observation"] == "test"
-def test_human_exit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
+def test_human_exit(dummy_env: SWEEnv, default_agent: DefaultAgent, tmp_path):
a = default_agent
a.model = PredeterminedTestModel(["```\nexit\n```"]) # type: ignore
a.setup(dummy_env, EmptyProblemStatement())
@@ -243,7 +243,7 @@ def test_human_exit(dummy_env: SWEEnv, default_agent: Agent, tmp_path):
assert r.action.strip() == "exit"
-def test_function_calling(dummy_env: SWEEnv, function_calling_agent: Agent, tmp_path):
+def test_function_calling(dummy_env: SWEEnv, function_calling_agent: DefaultAgent, tmp_path):
a = function_calling_agent
# Simulate a valid function call response from the model
valid_response = {