SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Reviewer: Simplify accept logic
Kilian Lieret7afcbb313a80
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex 8bc43b1f..ad731476 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -152,9 +152,7 @@ class ScoreRetryLoopConfig(BaseModel):reviewer_config: ReviewerConfigaccept_score: float- max_accepts: int- keep_top_scores: int- choice_variable: Literal["n_steps", "score"]+ max_accepts: int = 1max_attempts: int#: Minimal $ that need to be left in order for us to start a new attempt@@ -389,24 +387,33 @@ class ScoreRetryLoop(AbstractRetryLoop):if len(self._reviews) == 0:return Nonescores = [r.accept for r in self._reviews]- # IMPORTANT: Do not take s.info.model_stats.api_calls, because this is the cumulative cost over all attempts- steps = [s.model_stats.api_calls for s in self._submissions]- self.logger.debug(f"Scores: {scores}, n_steps: {steps}")- good_submissions = [i for i, s in enumerate(scores) if s >= self._config.accept_score]- self.logger.debug(f"Good submissions: {good_submissions} with scores: {[scores[i] for i in good_submissions]}")- if not good_submissions:- good_submissions = list(range(len(self._reviews)))- self.logger.debug("No good submissions found.")- good_submissions = sorted(good_submissions, key=lambda i: scores[i])[: self._config.keep_top_scores]- if self._config.choice_variable == "n_steps":- # Lowest number of steps- chosen_idx = min(good_submissions, key=lambda i: steps[i])- elif self._config.choice_variable == "score":- # Highest score- chosen_idx = max(good_submissions, key=lambda i: scores[i])+ self.logger.debug(f"Scores: {scores}")+ chosen_idx = max(range(len(scores)), key=scores.__getitem__)self.logger.info(f"Best submission: {chosen_idx}")return chosen_idx+ # def get_best(self) -> int | None:+ # if len(self._reviews) == 0:+ # return None+ # scores = [r.accept for r in self._reviews]+ # # IMPORTANT: Do not take s.info.model_stats.api_calls, because this is the cumulative cost over all attempts+ # steps = [s.model_stats.api_calls for s in self._submissions]+ # self.logger.debug(f"Scores: {scores}, n_steps: {steps}")+ # good_submissions = [i for i, s in enumerate(scores) if s >= self._config.accept_score]+ # self.logger.debug(f"Good submissions: {good_submissions} with scores: {[scores[i] for i in good_submissions]}")+ # if not good_submissions:+ # good_submissions = list(range(len(self._reviews)))+ # self.logger.debug("No good submissions found.")+ # good_submissions = sorted(good_submissions, key=lambda i: scores[i])[: self._config.keep_top_scores]+ # if self._config.choice_variable == "n_steps":+ # # Lowest number of steps+ # chosen_idx = min(good_submissions, key=lambda i: steps[i])+ # elif self._config.choice_variable == "score":+ # # Highest score+ # chosen_idx = max(good_submissions, key=lambda i: scores[i])+ # self.logger.info(f"Best submission: {chosen_idx}")+ # return chosen_idx+def get_retry_loop_from_config(config: RetryLoopConfig, problem_statement: ProblemStatement) -> ScoreRetryLoop:return config.get_retry_loop(problem_statement=problem_statement)