SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Feat/WIP: Chooser
Kilian Lieret37a124bf453b
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex dd374001..052a7118 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -30,7 +30,13 @@ from sweagent.agent.models import (get_model,)from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig-from sweagent.agent.reviewer import RetryLoopConfig, ReviewSubmission, ScoreRetryLoop, get_retry_loop_from_config+from sweagent.agent.reviewer import (+ ChooserRetryLoop,+ RetryLoopConfig,+ ReviewSubmission,+ ScoreRetryLoop,+ get_retry_loop_from_config,+)from sweagent.environment.swe_env import SWEEnvfrom sweagent.exceptions import (ContentPolicyViolationError,@@ -223,12 +229,12 @@ class RetryAgent(AbstractAgent):self._problem_statement: ProblemStatement | None = Noneself._env: SWEEnv | None = Noneself._output_dir: Path | None = None- self._rloop: ScoreRetryLoop | None = None+ self._rloop: ScoreRetryLoop | ChooserRetryLoop | None = None@propertydef _total_instance_stats(self) -> InstanceStats:assert self._rloop is not None- return self._total_instance_attempt_stats + self._rloop._model.stats+ return self._total_instance_attempt_stats + self._rloop.review_model_stats@classmethoddef from_config(cls, config: RetryAgentConfig) -> Self:@@ -303,29 +309,28 @@ class RetryAgent(AbstractAgent):self._attempt_data.append(self._agent.get_trajectory_data())self._total_instance_attempt_stats += self._agent.model.stats- def get_trajectory_data(self) -> dict[str, Any]:+ def get_trajectory_data(self, choose: bool) -> dict[str, Any]:"""Get all data that we save in .traj files."""assert self._rloop is not None- best_attempt_idx = self._rloop.get_best()- if best_attempt_idx is None:- data = {"info": {}}- else:- # Very important to deepcopy the best attempt data, because we will overwrite- # the stats with the total stats below.- data = {- "attempts": self._attempt_data,- **copy.deepcopy(self._attempt_data[best_attempt_idx]),- }- data["info"]["best_attempt_idx"] = best_attempt_idx- data["info"]["rloop_model_stats"] = self._rloop._model.stats.model_dump()- # Overwrite model stats with total stats- data["info"]["model_stats"] = self._total_instance_stats.model_dump()+ data = {+ "attempts": self._attempt_data,+ }++ if choose:+ best_attempt_idx = self._rloop.get_best()+ data |= copy.deepcopy(self._attempt_data[best_attempt_idx]) # type: ignore+ data["info"]["best_attempt_idx"] = best_attempt_idx+ data["info"]["rloop_model_stats"] = self._rloop.review_model_stats.model_dump()+ # Overwrite model stats with total stats+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()+ if isinstance(self._rloop, ChooserRetryLoop):+ data["info"]["chooser"] = self._rloop.chooser_output.model_dump() if self._rloop.chooser_output else {}return data- def save_trajectory(self) -> None:- data = self.get_trajectory_data()+ def save_trajectory(self, choose: bool) -> None:+ data = self.get_trajectory_data(choose=choose)assert self._traj_path is not Noneself._traj_path.write_text(json.dumps(data, indent=2))@@ -354,7 +359,7 @@ class RetryAgent(AbstractAgent):assert self._agent is not Nonewhile not step_output.done:step_output = self.step()- self.save_trajectory()+ self.save_trajectory(choose=False)if step_output.done:self._rloop.on_submit(ReviewSubmission(@@ -363,21 +368,22 @@ class RetryAgent(AbstractAgent):model_stats=self._agent.model.stats,))- self._agent.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore+ if isinstance(self._rloop, ScoreRetryLoop):+ self._agent.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignoreself._finalize_agent_run()- self.save_trajectory()+ self.save_trajectory(choose=False)if self._rloop.retry():assert self._env is not Noneself._next_attempt()step_output.done = False- self.save_trajectory() # call again after we finalized+ self.save_trajectory(choose=True) # call again after we finalizedself._chook.on_run_done(trajectory=self._agent.trajectory, info=self._agent.info)self.logger.info("Trajectory saved to %s", self._traj_path)# Here we want to return the "global" information (e.g., submission should# be the best submission instead of the last one, etc.), so we get it from the traj file- data = self.get_trajectory_data()+ data = self.get_trajectory_data(choose=True)return AgentRunResult(info=data["info"], trajectory=data["trajectory"])diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex aa040d28..c0faa2d1 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -5,6 +5,7 @@ solving the issue and to select the best solution.from __future__ import annotationsimport copy+import loggingimport refrom abc import ABC, abstractmethodfrom typing import Any, Literal@@ -21,6 +22,7 @@ from sweagent.agent.models import (get_model,)from sweagent.agent.problem_statement import ProblemStatement+from sweagent.tools.parsing import ActionParserfrom sweagent.tools.tools import ToolConfigfrom sweagent.types import AgentInfo, Trajectory, TrajectoryStepfrom sweagent.utils.log import get_logger@@ -61,6 +63,11 @@ class ReviewerResult(BaseModel):messages: list[dict[str, Any]]+class ChooserOutput(BaseModel):+ chosen_idx: int+ response: str++# --- INTERFACES ---@@ -111,6 +118,15 @@ class AbstractRetryLoop(ABC):# --- CONFIGS ---+class ChooserConfig(BaseModel):+ model: ModelConfig+ system_template: str+ instance_template: str+ submission_template: str+ max_len_submission: int = 5000+ # summarizer: SummarizerConfig | None = None++class TrajFormatterConfig(BaseModel):#: Filter the following actions from the trajectoryfilter: list[str] = []@@ -146,6 +162,26 @@ class ReviewerConfig(BaseModel):return Reviewer(self, model)+class ChooserRetryLoopConfig(BaseModel):+ type: Literal["chooser"] = "chooser"+ chooser_config: ChooserConfig++ max_attempts: int+ min_budget_for_new_attempt: float = 0.0+ """Minimal $ that need to be left in order for us to start a new attempt.+ If set to 0: Always.+ """++ cost_limit: float+ """The maximum cost to spend on all attempts. Does not include cost of choosing.+ """++ model_config = ConfigDict(extra="forbid")++ def get_retry_loop(self, problem_statement: ProblemStatement) -> ChooserRetryLoop:+ return ChooserRetryLoop(self, problem_statement)++class ScoreRetryLoopConfig(BaseModel):"""The configuration for the review loop"""@@ -188,6 +224,51 @@ RetryLoopConfig = ScoreRetryLoopConfig# --- IMPLEMENTATIONS ---+class Chooser:+ def __init__(self, config: ChooserConfig):+ self.config = config+ self.model = get_model(config.model, ToolConfig(parse_function=ActionParser()))+ self.model.logger.setLevel(logging.WARNING) # type: ignore+ self.logger = get_logger("chooser", emoji="🧠")+ # self.summarizer = Summarizer(config.summarizer, self.model) if config.summarizer else None++ def interpret(self, response: str) -> int:+ # Use regex to extract the last number of the response+ try:+ return int(re.findall(r"\d+", response)[-1])+ except Exception as e:+ self.logger.error(f"Error interpreting response: {e}")+ return 0++ def format_submission(self, problem_statement: str, submission: ReviewSubmission) -> str:+ if (+ submission.info.get("submission") is None+ or len(submission.info.get("submission", "")) > self.config.max_len_submission > 0 # type: ignore+ ):+ return "Solution invalid."+ return Template(self.config.submission_template).render(+ **submission.to_format_dict(),+ # summary=self.summarizer.summarize(problem_statement, submission.trajectory) if self.summarizer else "",+ )++ def build_messages(self, problem_statement: str, input: ReviewSubmission) -> list[dict[str, Any]]:+ instance_message = Template(self.config.instance_template).render(+ problem_statement=problem_statement,+ **input.to_format_dict(),+ )+ self.logger.debug(f"MODEL INPUT (user)\n{instance_message}")+ return [+ {"role": "system", "content": self.config.system_template},+ {"role": "user", "content": instance_message},+ ]++ def choose(self, problem_statement: str, input: ReviewSubmission) -> ChooserOutput:+ messages = self.build_messages(problem_statement, input)+ response = self.model.query(messages)["message"] # type: ignore+ chosen_idx = self.interpret(response)+ return ChooserOutput(chosen_idx=chosen_idx, response=response)++class Reviewer(AbstractReviewer):def __init__(self, config: ReviewerConfig, model):self._config = config@@ -312,6 +393,67 @@ class TrajectoryFormatter:)+class ChooserRetryLoop(AbstractRetryLoop):+ def __init__(self, config: ChooserRetryLoopConfig, problem_statement: ProblemStatement):+ self._config = config+ self._problem_statement = problem_statement+ self._chooser = Chooser(config.chooser_config)+ self._submissions: list[ReviewSubmission] = []+ self._n_consec_exit_cost: int = 0+ self.logger = get_logger("chooser_loop", emoji="🔄")+ self.chooser_output: ChooserOutput | None = None++ @property+ def _total_stats(self) -> InstanceStats:+ return sum((s.model_stats for s in self._submissions), start=InstanceStats())++ @property+ def review_model_stats(self) -> InstanceStats:+ return InstanceStats()++ @property+ def _n_attempts(self) -> int:+ return len(self._submissions)++ def on_submit(self, submission: ReviewSubmission) -> None:+ self._submissions.append(submission)++ def retry(self) -> bool:+ stat_str = f"n_samples={self._n_attempts}"+ if self._total_stats.instance_cost > self._config.cost_limit > 0:+ self.logger.info(+ f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_stats.instance_cost}) "+ f"exceeds cost limit ({self._config.cost_limit})"+ )+ return False++ if self._n_attempts >= self._config.max_attempts > 0:+ self.logger.info(f"Exiting retry loop ({stat_str}): max_attempts={self._config.max_attempts} reached")+ return False++ remaining_budget = self._config.cost_limit - self._total_stats.instance_cost+ if self._config.min_budget_for_new_attempt > 0 and remaining_budget < self._config.min_budget_for_new_attempt:+ msg = (+ f"Exiting retry loop ({stat_str}): Not enough budget left for a new attempt "+ f"({remaining_budget} remaining, {self._config.min_budget_for_new_attempt} required)"+ )+ self.logger.info(msg)+ return False++ return True++ def get_best(self) -> int | None:+ """Important note: This is cached. Only call this at the end."""+ if self.chooser_output is not None:+ return self.chooser_output.chosen_idx+ if len(self._submissions) == 0:+ return None+ self.chooser_output = self._chooser.choose(+ self._problem_statement.get_problem_statement(), self._submissions[-1]+ )+ return self.chooser_output.chosen_idx++class ScoreRetryLoop(AbstractRetryLoop):def __init__(self,@@ -334,6 +476,10 @@ class ScoreRetryLoop(AbstractRetryLoop):# Properties# ----------+ @property+ def review_model_stats(self) -> InstanceStats:+ return self._model.stats+@propertydef reviews(self) -> list[ReviewerResult]:return self._reviews@@ -432,5 +578,7 @@ class ScoreRetryLoop(AbstractRetryLoop):# return chosen_idx-def get_retry_loop_from_config(config: RetryLoopConfig, problem_statement: ProblemStatement) -> ScoreRetryLoop:+def get_retry_loop_from_config(+ config: RetryLoopConfig, problem_statement: ProblemStatement+) -> ScoreRetryLoop | ChooserRetryLoop:return config.get_retry_loop(problem_statement=problem_statement)