SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Enh(retry agent): Include reviews cost in retry decision
Kilian Lieret54bd584c2758
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex a0b1c27c..76cadb15 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -214,7 +214,10 @@ class RetryAgent(AbstractAgent):self.logger = get_logger("swea-agent", emoji="🤠")self._agent: DefaultAgent | None = Noneself._attempt_data: list[dict[str, Any]] = []- self._total_instance_stats = InstanceStats()+ self._total_instance_attempt_stats = InstanceStats()+ """Note that total_instance_attempt_stats only accumulates the states of the sub-agent,+ not the reviewer. Use self._total_instance_stats for the total stats.+ """self._chook = CombinedAgentHook()self._traj_path: Path | None = Noneself._problem_statement: ProblemStatement | None = None@@ -222,6 +225,11 @@ class RetryAgent(AbstractAgent):self._output_dir: Path | None = Noneself._rloop: ScoreRetryLoop | None = None+ @property+ def _total_instance_stats(self) -> InstanceStats:+ assert self._rloop is not None+ return self._total_instance_attempt_stats + self._rloop._model.stats+@classmethoddef from_config(cls, config: RetryAgentConfig) -> Self:return cls(config)@@ -236,7 +244,7 @@ class RetryAgent(AbstractAgent):"""Setup the retry agent for a new problem instance.This is mostly a bookkeeping step."""- self._total_instance_stats = InstanceStats()+ self._total_instance_attempt_stats = InstanceStats()self._problem_statement = problem_statementself._traj_path = output_dir / (self._problem_statement.id + ".traj")self._env = env@@ -286,7 +294,7 @@ class RetryAgent(AbstractAgent):assert self._agent is not Noneself._agent.save_trajectory()self._attempt_data.append(self._agent.get_trajectory_data())- self._total_instance_stats += self._agent.model.stats+ self._total_instance_attempt_stats += self._agent.model.statsdef get_trajectory_data(self) -> dict[str, Any]:"""Get all data that we save in .traj files."""@@ -301,9 +309,9 @@ class RetryAgent(AbstractAgent):**self._attempt_data[best_attempt_idx],}data["info"]["best_attempt_idx"] = best_attempt_idx- data["info"]["rloop_model_stats"] = self._rloop.model_stats.model_dump()+ data["info"]["rloop_model_stats"] = self._rloop._model.stats.model_dump()# Overwrite model stats with total stats- data["info"]["model_stats"] = (self._total_instance_stats + self._rloop.model_stats).model_dump()+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()return datadiff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex 41892236..bd3cf114 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -342,12 +342,8 @@ class ScoreRetryLoop(AbstractRetryLoop):return sum(r.accept >= self._config.accept_score for r in self._reviews)@property- def model_stats(self) -> InstanceStats:- return self._model.stats-- @property- def _total_attempt_stats(self) -> InstanceStats:- return sum((s.model_stats for s in self._submissions), start=InstanceStats())+ def _total_stats(self) -> InstanceStats:+ return sum((s.model_stats for s in self._submissions), start=InstanceStats()) + self._model.stats# -------@@ -369,9 +365,9 @@ class ScoreRetryLoop(AbstractRetryLoop):max_score = max([r.accept for r in self._reviews], default=-100.0)stat_str = f"n_samples={self._n_attempts}, max_score={max_score}, n_accepted={self._n_accepted}"- if self._total_attempt_stats.instance_cost > self._config.cost_limit > 0:+ if self._total_stats.instance_cost > self._config.cost_limit > 0:self.logger.info(- f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_attempt_stats.instance_cost}) "+ f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_stats.instance_cost}) "f"exceeds cost limit ({self._config.cost_limit})")return False@@ -384,7 +380,7 @@ class ScoreRetryLoop(AbstractRetryLoop):self.logger.info(f"Exiting retry loop ({stat_str}): max_accepts={self._config.max_accepts} reached")return False- remaining_budget = self._config.cost_limit - self._total_attempt_stats.instance_cost+ remaining_budget = self._config.cost_limit - self._total_stats.instance_costif self._config.min_budget_for_new_attempt > 0 and remaining_budget < self._config.min_budget_for_new_attempt:msg = (f"Exiting retry loop ({stat_str}): Not enough budget left for a new attempt "