SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
WIP: Copy over configs from evaluate_v2
Kilian Lieret287bfee9dc6f
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex 4adcd52e..d0b15bb8 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -107,22 +107,28 @@ class AbstractRetryLoop(ABC):# --- CONFIGS ---+class TrajFormatterConfig(BaseModel):+ #: Filter the following actions from the trajectory+ filter: list[str] = []+ #: Filter outputs from the following actions from the trajectory+ output_filter: list[str] = []+ #: Format of the trajectory item+ item_template: str = "Model: {{response}}\n\nObservation: {{observation}}"+ only_show_last_n_output: int = 0++class ReviewerConfig(BaseModel):"""The configuration for the reviewer"""- output_type: Literal["bool", "float"] = "bool"+ output_type: Literal["bool", "float"] = "float"system_template: strinstance_template: str#: If a submission autosubmits because of total cost or a similar exit status,#: it will be desk rejectedreject_exit_status: bool = True- #: Filter the following actions from the trajectory- traj_filter: list[str] = []- #: Filter outputs from the following actions from the trajectory- traj_output_filter: list[str] = []- #: Format of the trajectory item- traj_item_template: str = "Model: {response}\n\nObservation: {observation}"- filter_failed_edits: bool = False+ traj_formatter: TrajFormatterConfig++ type: Literal["reviewer"] = "reviewer"class BinaryReviewerConfig(BaseModel):@@ -130,13 +136,9 @@ class BinaryReviewerConfig(BaseModel):system_template: strinstance_template: str- #: Filter the following actions from the trajectory- traj_filter: list[str] = []- #: Format of the trajectory item- traj_item_template: str = "Model: {response}\n\nObservation: {observation}"- traj_output_filter: list[str] = []- traj_filter_failed_edits: bool = False- traj_only_show_last_n_output: int = 0+ traj_formatter: TrajFormatterConfig++ type: Literal["binary_reviewer"] = "binary_reviewer"class GTCConfig(BaseModel):@@ -196,16 +198,11 @@ RetryLoopConfig = ScoreRetryLoopConfig# --- IMPLEMENTATIONS ----class Reviewer(AbstractReviewer):+class Reviewer:def __init__(self, config: ReviewerConfig, model):self._config = configself._model = model- self._traj_formatter = TrajectoryFormatter(- traj_filter=config.traj_filter,- traj_item_template=config.traj_item_template,- traj_output_filter=config.traj_output_filter,- filter_failed_edits=config.filter_failed_edits,- )+ self._traj_formatter = TrajectoryFormatter(config=config.traj_formatter)self.logger = get_logger("reviewer", emoji="🧑⚖️")def format_messages(self, instance: ProblemStatement, submission: ReviewSubmission):@@ -241,7 +238,10 @@ class Reviewer(AbstractReviewer):if number:return float(number.group(0))else:- self.logger.warning("Could not interpret response: %s, will reject submission.", response)+ self.logger.warning(+ "Could not interpret response: %s, will reject submission.",+ response,+ )return 0.0raise ValueError@@ -258,43 +258,33 @@ class Reviewer(AbstractReviewer):messages = self.format_messages(instance, submission)answer = self._model.query(messages, temperature=0.0)["message"]accept = self.interpret(answer)- accept_emoji = "✅" if accept else "❌"- self.logger.info(f"{accept_emoji}\n{answer}")+ self.logger.info(answer)return ReviewerResult(accept=accept, output=answer, messages=messages)# todo: Couldn't I just replace the whole thing with Jinja templates?++class TrajectoryFormatter:def __init__(self,- *,- traj_filter: list[str] | None = None,- traj_output_filter: list[str] | None = None,- traj_item_template: str = "Model: {{response}}\n\nObservation: {{observation}}",- filter_failed_edits: bool = False,- only_show_last_n_output: int = 0,+ config: TrajFormatterConfig,):"""Formats trajectories for the use in prompts"""- self._traj_filter = traj_filter or []- self._traj_output_filter = traj_output_filter or []- self._traj_item_template = traj_item_template- self._filter_failed_edits = filter_failed_edits- self._only_show_last_n_output = only_show_last_n_output+ self._config = configdef _include_step(self, item: TrajectoryStep) -> bool:action = item["action"].strip()- for f in self._traj_filter:+ for f in self._config.filter:if action.startswith(f):return False- if self._filter_failed_edits and "Your proposed edit has introduced new syntax error(s)" in item["observation"]:- return Falsereturn Truedef _include_step_output(self, item: TrajectoryStep, i_step: int, n_steps: int) -> bool:- if self._only_show_last_n_output > 0 and i_step < n_steps - self._only_show_last_n_output:+ if self._config.only_show_last_n_output > 0 and i_step < n_steps - self._config.only_show_last_n_output:return Falseaction = item["action"].strip()- for f in self._traj_output_filter:+ for f in self._config.output_filter:if action.startswith(f):return Falsereturn True@@ -303,7 +293,7 @@ class TrajectoryFormatter:step = copy.deepcopy(step)if not self._include_step_output(step, i_step, n_steps=n_steps):step["observation"] = "[Output omitted]"- return Template(self._traj_item_template).render(+ return Template(self._config.item_template).render(**step,i_step=i_step,i_traj=i_traj,@@ -319,17 +309,11 @@ class TrajectoryFormatter:)-class BinaryReviewer(AbstractBinaryReviewer):- def __init__(self, config: BinaryReviewerConfig, model: AbstractModel):+class BinaryReviewer:+ def __init__(self, config: BinaryReviewerConfig, model: LiteLLMModel):self._config = configself._model = model- self._traj_formatter = TrajectoryFormatter(- traj_filter=config.traj_filter,- traj_item_template=config.traj_item_template,- traj_output_filter=config.traj_output_filter,- filter_failed_edits=config.traj_filter_failed_edits,- only_show_last_n_output=config.traj_only_show_last_n_output,- )+ self._traj_formatter = TrajectoryFormatter(config=config.traj_formatter)self.logger = get_logger("binary_reviewer", emoji="⚖️")def format_messages(self, instance: ProblemStatement, sub1: ReviewSubmission, sub2: ReviewSubmission):@@ -339,13 +323,15 @@ class BinaryReviewer(AbstractBinaryReviewer):"problem_statement": instance.get_problem_statement(),**instance.get_extra_fields(),}- user_message = Template(self._config.instance_template).render(+ format_dict = {**ps_format_dict,**sub1.to_format_dict(suffix="1"),**sub2.to_format_dict(suffix="2"),- traj1=self._traj_formatter.format_trajectory(sub1.trajectory, i_traj=1),- traj2=self._traj_formatter.format_trajectory(sub2.trajectory, i_traj=2),- )+ "traj1": self._traj_formatter.format_trajectory(sub1.trajectory, i_traj=1),+ "traj2": self._traj_formatter.format_trajectory(sub2.trajectory, i_traj=2),+ }+ # print(format_dict)+ user_message = Template(self._config.instance_template).render(**format_dict)self.logger.debug(f"MODEL INPUT (user)\n{user_message}")return [{"role": "system", "content": system_message},@@ -370,7 +356,8 @@ class BinaryReviewer(AbstractBinaryReviewer):elif "second" in last_line.lower():return (1, confidence)self.logger.warning(- "Could not interpret response: %s, will choose first submission with confidence 0.0.", response+ "Could not interpret response: %s, will choose first submission with confidence 0.0.",+ response,)return (0, 0.0)