Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Enh(retry agent): Include reviews cost in retry decision

Kilian Lieret54bd584c2758
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index a0b1c27c..76cadb15 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -214,7 +214,10 @@ class RetryAgent(AbstractAgent):
self.logger = get_logger("swea-agent", emoji="🤠")
self._agent: DefaultAgent | None = None
self._attempt_data: list[dict[str, Any]] = []
- self._total_instance_stats = InstanceStats()
+ self._total_instance_attempt_stats = InstanceStats()
+ """Note that total_instance_attempt_stats only accumulates the states of the sub-agent,
+ not the reviewer. Use self._total_instance_stats for the total stats.
+ """
self._chook = CombinedAgentHook()
self._traj_path: Path | None = None
self._problem_statement: ProblemStatement | None = None
@@ -222,6 +225,11 @@ class RetryAgent(AbstractAgent):
self._output_dir: Path | None = None
self._rloop: ScoreRetryLoop | None = None
+ @property
+ def _total_instance_stats(self) -> InstanceStats:
+ assert self._rloop is not None
+ return self._total_instance_attempt_stats + self._rloop._model.stats
+
@classmethod
def from_config(cls, config: RetryAgentConfig) -> Self:
return cls(config)
@@ -236,7 +244,7 @@ class RetryAgent(AbstractAgent):
"""Setup the retry agent for a new problem instance.
This is mostly a bookkeeping step.
"""
- self._total_instance_stats = InstanceStats()
+ self._total_instance_attempt_stats = InstanceStats()
self._problem_statement = problem_statement
self._traj_path = output_dir / (self._problem_statement.id + ".traj")
self._env = env
@@ -286,7 +294,7 @@ class RetryAgent(AbstractAgent):
assert self._agent is not None
self._agent.save_trajectory()
self._attempt_data.append(self._agent.get_trajectory_data())
- self._total_instance_stats += self._agent.model.stats
+ self._total_instance_attempt_stats += self._agent.model.stats
def get_trajectory_data(self) -> dict[str, Any]:
"""Get all data that we save in .traj files."""
@@ -301,9 +309,9 @@ class RetryAgent(AbstractAgent):
**self._attempt_data[best_attempt_idx],
}
data["info"]["best_attempt_idx"] = best_attempt_idx
- data["info"]["rloop_model_stats"] = self._rloop.model_stats.model_dump()
+ data["info"]["rloop_model_stats"] = self._rloop._model.stats.model_dump()
# Overwrite model stats with total stats
- data["info"]["model_stats"] = (self._total_instance_stats + self._rloop.model_stats).model_dump()
+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()
return data
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index 41892236..bd3cf114 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -342,12 +342,8 @@ class ScoreRetryLoop(AbstractRetryLoop):
return sum(r.accept >= self._config.accept_score for r in self._reviews)
@property
- def model_stats(self) -> InstanceStats:
- return self._model.stats
-
- @property
- def _total_attempt_stats(self) -> InstanceStats:
- return sum((s.model_stats for s in self._submissions), start=InstanceStats())
+ def _total_stats(self) -> InstanceStats:
+ return sum((s.model_stats for s in self._submissions), start=InstanceStats()) + self._model.stats
# -------
@@ -369,9 +365,9 @@ class ScoreRetryLoop(AbstractRetryLoop):
max_score = max([r.accept for r in self._reviews], default=-100.0)
stat_str = f"n_samples={self._n_attempts}, max_score={max_score}, n_accepted={self._n_accepted}"
- if self._total_attempt_stats.instance_cost > self._config.cost_limit > 0:
+ if self._total_stats.instance_cost > self._config.cost_limit > 0:
self.logger.info(
- f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_attempt_stats.instance_cost}) "
+ f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_stats.instance_cost}) "
f"exceeds cost limit ({self._config.cost_limit})"
)
return False
@@ -384,7 +380,7 @@ class ScoreRetryLoop(AbstractRetryLoop):
self.logger.info(f"Exiting retry loop ({stat_str}): max_accepts={self._config.max_accepts} reached")
return False
- remaining_budget = self._config.cost_limit - self._total_attempt_stats.instance_cost
+ remaining_budget = self._config.cost_limit - self._total_stats.instance_cost
if self._config.min_budget_for_new_attempt > 0 and remaining_budget < self._config.min_budget_for_new_attempt:
msg = (
f"Exiting retry loop ({stat_str}): Not enough budget left for a new attempt "