Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Feat/WIP: Chooser

Kilian Lieret37a124bf453b
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index dd374001..052a7118 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -30,7 +30,13 @@ from sweagent.agent.models import (
get_model,
)
from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig
-from sweagent.agent.reviewer import RetryLoopConfig, ReviewSubmission, ScoreRetryLoop, get_retry_loop_from_config
+from sweagent.agent.reviewer import (
+ ChooserRetryLoop,
+ RetryLoopConfig,
+ ReviewSubmission,
+ ScoreRetryLoop,
+ get_retry_loop_from_config,
+)
from sweagent.environment.swe_env import SWEEnv
from sweagent.exceptions import (
ContentPolicyViolationError,
@@ -223,12 +229,12 @@ class RetryAgent(AbstractAgent):
self._problem_statement: ProblemStatement | None = None
self._env: SWEEnv | None = None
self._output_dir: Path | None = None
- self._rloop: ScoreRetryLoop | None = None
+ self._rloop: ScoreRetryLoop | ChooserRetryLoop | None = None
@property
def _total_instance_stats(self) -> InstanceStats:
assert self._rloop is not None
- return self._total_instance_attempt_stats + self._rloop._model.stats
+ return self._total_instance_attempt_stats + self._rloop.review_model_stats
@classmethod
def from_config(cls, config: RetryAgentConfig) -> Self:
@@ -303,29 +309,28 @@ class RetryAgent(AbstractAgent):
self._attempt_data.append(self._agent.get_trajectory_data())
self._total_instance_attempt_stats += self._agent.model.stats
- def get_trajectory_data(self) -> dict[str, Any]:
+ def get_trajectory_data(self, choose: bool) -> dict[str, Any]:
"""Get all data that we save in .traj files."""
assert self._rloop is not None
- best_attempt_idx = self._rloop.get_best()
- if best_attempt_idx is None:
- data = {"info": {}}
- else:
- # Very important to deepcopy the best attempt data, because we will overwrite
- # the stats with the total stats below.
- data = {
- "attempts": self._attempt_data,
- **copy.deepcopy(self._attempt_data[best_attempt_idx]),
- }
- data["info"]["best_attempt_idx"] = best_attempt_idx
- data["info"]["rloop_model_stats"] = self._rloop._model.stats.model_dump()
- # Overwrite model stats with total stats
- data["info"]["model_stats"] = self._total_instance_stats.model_dump()
+ data = {
+ "attempts": self._attempt_data,
+ }
+
+ if choose:
+ best_attempt_idx = self._rloop.get_best()
+ data |= copy.deepcopy(self._attempt_data[best_attempt_idx]) # type: ignore
+ data["info"]["best_attempt_idx"] = best_attempt_idx
+ data["info"]["rloop_model_stats"] = self._rloop.review_model_stats.model_dump()
+ # Overwrite model stats with total stats
+ data["info"]["model_stats"] = self._total_instance_stats.model_dump()
+ if isinstance(self._rloop, ChooserRetryLoop):
+ data["info"]["chooser"] = self._rloop.chooser_output.model_dump() if self._rloop.chooser_output else {}
return data
- def save_trajectory(self) -> None:
- data = self.get_trajectory_data()
+ def save_trajectory(self, choose: bool) -> None:
+ data = self.get_trajectory_data(choose=choose)
assert self._traj_path is not None
self._traj_path.write_text(json.dumps(data, indent=2))
@@ -354,7 +359,7 @@ class RetryAgent(AbstractAgent):
assert self._agent is not None
while not step_output.done:
step_output = self.step()
- self.save_trajectory()
+ self.save_trajectory(choose=False)
if step_output.done:
self._rloop.on_submit(
ReviewSubmission(
@@ -363,21 +368,22 @@ class RetryAgent(AbstractAgent):
model_stats=self._agent.model.stats,
)
)
- self._agent.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore
+ if isinstance(self._rloop, ScoreRetryLoop):
+ self._agent.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore
self._finalize_agent_run()
- self.save_trajectory()
+ self.save_trajectory(choose=False)
if self._rloop.retry():
assert self._env is not None
self._next_attempt()
step_output.done = False
- self.save_trajectory() # call again after we finalized
+ self.save_trajectory(choose=True) # call again after we finalized
self._chook.on_run_done(trajectory=self._agent.trajectory, info=self._agent.info)
self.logger.info("Trajectory saved to %s", self._traj_path)
# Here we want to return the "global" information (e.g., submission should
# be the best submission instead of the last one, etc.), so we get it from the traj file
- data = self.get_trajectory_data()
+ data = self.get_trajectory_data(choose=True)
return AgentRunResult(info=data["info"], trajectory=data["trajectory"])
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index aa040d28..c0faa2d1 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -5,6 +5,7 @@ solving the issue and to select the best solution.
from __future__ import annotations
import copy
+import logging
import re
from abc import ABC, abstractmethod
from typing import Any, Literal
@@ -21,6 +22,7 @@ from sweagent.agent.models import (
get_model,
)
from sweagent.agent.problem_statement import ProblemStatement
+from sweagent.tools.parsing import ActionParser
from sweagent.tools.tools import ToolConfig
from sweagent.types import AgentInfo, Trajectory, TrajectoryStep
from sweagent.utils.log import get_logger
@@ -61,6 +63,11 @@ class ReviewerResult(BaseModel):
messages: list[dict[str, Any]]
+class ChooserOutput(BaseModel):
+ chosen_idx: int
+ response: str
+
+
# --- INTERFACES ---
@@ -111,6 +118,15 @@ class AbstractRetryLoop(ABC):
# --- CONFIGS ---
+class ChooserConfig(BaseModel):
+ model: ModelConfig
+ system_template: str
+ instance_template: str
+ submission_template: str
+ max_len_submission: int = 5000
+ # summarizer: SummarizerConfig | None = None
+
+
class TrajFormatterConfig(BaseModel):
#: Filter the following actions from the trajectory
filter: list[str] = []
@@ -146,6 +162,26 @@ class ReviewerConfig(BaseModel):
return Reviewer(self, model)
+class ChooserRetryLoopConfig(BaseModel):
+ type: Literal["chooser"] = "chooser"
+ chooser_config: ChooserConfig
+
+ max_attempts: int
+ min_budget_for_new_attempt: float = 0.0
+ """Minimal $ that need to be left in order for us to start a new attempt.
+ If set to 0: Always.
+ """
+
+ cost_limit: float
+ """The maximum cost to spend on all attempts. Does not include cost of choosing.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ def get_retry_loop(self, problem_statement: ProblemStatement) -> ChooserRetryLoop:
+ return ChooserRetryLoop(self, problem_statement)
+
+
class ScoreRetryLoopConfig(BaseModel):
"""The configuration for the review loop"""
@@ -188,6 +224,51 @@ RetryLoopConfig = ScoreRetryLoopConfig
# --- IMPLEMENTATIONS ---
+class Chooser:
+ def __init__(self, config: ChooserConfig):
+ self.config = config
+ self.model = get_model(config.model, ToolConfig(parse_function=ActionParser()))
+ self.model.logger.setLevel(logging.WARNING) # type: ignore
+ self.logger = get_logger("chooser", emoji="🧠")
+ # self.summarizer = Summarizer(config.summarizer, self.model) if config.summarizer else None
+
+ def interpret(self, response: str) -> int:
+ # Use regex to extract the last number of the response
+ try:
+ return int(re.findall(r"\d+", response)[-1])
+ except Exception as e:
+ self.logger.error(f"Error interpreting response: {e}")
+ return 0
+
+ def format_submission(self, problem_statement: str, submission: ReviewSubmission) -> str:
+ if (
+ submission.info.get("submission") is None
+ or len(submission.info.get("submission", "")) > self.config.max_len_submission > 0 # type: ignore
+ ):
+ return "Solution invalid."
+ return Template(self.config.submission_template).render(
+ **submission.to_format_dict(),
+ # summary=self.summarizer.summarize(problem_statement, submission.trajectory) if self.summarizer else "",
+ )
+
+ def build_messages(self, problem_statement: str, input: ReviewSubmission) -> list[dict[str, Any]]:
+ instance_message = Template(self.config.instance_template).render(
+ problem_statement=problem_statement,
+ **input.to_format_dict(),
+ )
+ self.logger.debug(f"MODEL INPUT (user)\n{instance_message}")
+ return [
+ {"role": "system", "content": self.config.system_template},
+ {"role": "user", "content": instance_message},
+ ]
+
+ def choose(self, problem_statement: str, input: ReviewSubmission) -> ChooserOutput:
+ messages = self.build_messages(problem_statement, input)
+ response = self.model.query(messages)["message"] # type: ignore
+ chosen_idx = self.interpret(response)
+ return ChooserOutput(chosen_idx=chosen_idx, response=response)
+
+
class Reviewer(AbstractReviewer):
def __init__(self, config: ReviewerConfig, model):
self._config = config
@@ -312,6 +393,67 @@ class TrajectoryFormatter:
)
+class ChooserRetryLoop(AbstractRetryLoop):
+ def __init__(self, config: ChooserRetryLoopConfig, problem_statement: ProblemStatement):
+ self._config = config
+ self._problem_statement = problem_statement
+ self._chooser = Chooser(config.chooser_config)
+ self._submissions: list[ReviewSubmission] = []
+ self._n_consec_exit_cost: int = 0
+ self.logger = get_logger("chooser_loop", emoji="🔄")
+ self.chooser_output: ChooserOutput | None = None
+
+ @property
+ def _total_stats(self) -> InstanceStats:
+ return sum((s.model_stats for s in self._submissions), start=InstanceStats())
+
+ @property
+ def review_model_stats(self) -> InstanceStats:
+ return InstanceStats()
+
+ @property
+ def _n_attempts(self) -> int:
+ return len(self._submissions)
+
+ def on_submit(self, submission: ReviewSubmission) -> None:
+ self._submissions.append(submission)
+
+ def retry(self) -> bool:
+ stat_str = f"n_samples={self._n_attempts}"
+ if self._total_stats.instance_cost > self._config.cost_limit > 0:
+ self.logger.info(
+ f"Exiting retry loop ({stat_str}): Total attempt cost ({self._total_stats.instance_cost}) "
+ f"exceeds cost limit ({self._config.cost_limit})"
+ )
+ return False
+
+ if self._n_attempts >= self._config.max_attempts > 0:
+ self.logger.info(f"Exiting retry loop ({stat_str}): max_attempts={self._config.max_attempts} reached")
+ return False
+
+ remaining_budget = self._config.cost_limit - self._total_stats.instance_cost
+ if self._config.min_budget_for_new_attempt > 0 and remaining_budget < self._config.min_budget_for_new_attempt:
+ msg = (
+ f"Exiting retry loop ({stat_str}): Not enough budget left for a new attempt "
+ f"({remaining_budget} remaining, {self._config.min_budget_for_new_attempt} required)"
+ )
+ self.logger.info(msg)
+ return False
+
+ return True
+
+ def get_best(self) -> int | None:
+ """Important note: This is cached. Only call this at the end."""
+ if self.chooser_output is not None:
+ return self.chooser_output.chosen_idx
+ if len(self._submissions) == 0:
+ return None
+ self.chooser_output = self._chooser.choose(
+ self._problem_statement.get_problem_statement(), self._submissions[-1]
+ )
+ return self.chooser_output.chosen_idx
+
+
class ScoreRetryLoop(AbstractRetryLoop):
def __init__(
self,
@@ -334,6 +476,10 @@ class ScoreRetryLoop(AbstractRetryLoop):
# Properties
# ----------
+ @property
+ def review_model_stats(self) -> InstanceStats:
+ return self._model.stats
+
@property
def reviews(self) -> list[ReviewerResult]:
return self._reviews
@@ -432,5 +578,7 @@ class ScoreRetryLoop(AbstractRetryLoop):
# return chosen_idx
-def get_retry_loop_from_config(config: RetryLoopConfig, problem_statement: ProblemStatement) -> ScoreRetryLoop:
+def get_retry_loop_from_config(
+ config: RetryLoopConfig, problem_statement: ProblemStatement
+) -> ScoreRetryLoop | ChooserRetryLoop:
return config.get_retry_loop(problem_statement=problem_statement)