SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Fix: Proper bookkeeping of attempts for review & retry
Kilian Lieretccefa72beca1
diff --git a/config/function_calling_anthropic_replace_late_repro_review_submit_retry.yaml b/config/function_calling_anthropic_replace_late_repro_review_submit_retry.yamlindex a125ee4d..33e581fd 100644--- a/config/function_calling_anthropic_replace_late_repro_review_submit_retry.yaml+++ b/config/function_calling_anthropic_replace_late_repro_review_submit_retry.yaml@@ -312,7 +312,7 @@ agent:This is the final patch of investigation 1:```- {submission1}+ {{submission1}}```========================diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex d28cedb7..f439a339 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -27,7 +27,7 @@ from sweagent.agent.models import (get_model,)from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig-from sweagent.agent.reviewer import ReviewLoopConfig, get_review_loop_from_config+from sweagent.agent.reviewer import AbstractReviewLoop, ReviewLoopConfig, get_review_loop_from_configfrom sweagent.environment.swe_env import SWEEnvfrom sweagent.exceptions import ContextWindowExceededError, CostLimitExceededError, FormatErrorfrom sweagent.tools.parsing import (@@ -171,6 +171,7 @@ class Agent:#: Number of attempts to solve the issue when using a review loopself._i_attempt: int = 0+ self._rloop: AbstractReviewLoop | None = None#: The following three attributes collect the information about how the agent#: solved the problem.@@ -264,6 +265,14 @@ class Agent:return messages+ @property+ def attempt_model_stats(self) -> InstanceStats:+ """Model stats of the current attempt"""+ total_stats = self.model.stats+ for attempt_idx in range(self._i_attempt):+ total_stats -= InstanceStats.model_validate(self._info_by_attempt[attempt_idx]["model_stats"]) # type: ignore+ return total_stats+# Methods# -------@@ -282,9 +291,6 @@ class Agent:formatting the system message and adding demonstrations to the history.This method is called by `self.run`.-- Args:- instance_args: Arguments for the instance"""self._problem_statement = problem_statementself._env = env@@ -298,8 +304,10 @@ class Agent:self._trajectory_by_attempt = defaultdict(list)self._info_by_attempt = defaultdict(dict) # type: ignoreself._forwarded_vars = {}- # if self._rloop is not None:- # self._forwarded_vars = self._rloop.get_forwarded_vars()+ if self.review_loop_config is not None:+ self._rloop = get_review_loop_from_config(self.review_loop_config, problem_statement, self.model)+ if self._rloop is not None:+ self._forwarded_vars = self._rloop.get_forwarded_vars()self._chook.on_tools_installation_started()self.tools.install(self._env)self.setup_attempt()@@ -322,6 +330,7 @@ class Agent:self.tools.install(self._env)else:self._env.reset()+ assert self._problem_statement is not Noneself._env.set_env_variables({"PROBLEM_STATEMENT": self._problem_statement.get_problem_statement()})self.add_system_message_to_history()self.add_demonstrations_to_history()@@ -420,7 +429,12 @@ class Agent:message = "\n".join(messages)self.logger.info(f"🤖 MODEL INPUT\n{message}")- history_item = {"role": "user", "content": message, "agent": self.name, "message_type": "observation"}+ history_item: dict[str, Any] = {+ "role": "user",+ "content": message,+ "agent": self.name,+ "message_type": "observation",+ }if tool_call_ids:assert len(tool_call_ids) == 1, "This should be ensured by the FunctionCalling parse method"history_item["role"] = "tool"@@ -493,10 +507,18 @@ class Agent:attempt_data["environment"] = self._env.namereturn attempt_data- data = {- **get_attempt_data(self._i_attempt),- "attempts": [get_attempt_data(i) for i in range(self._i_attempt + 1)],- }+ if self._rloop is not None:+ best_attempt_idx = self._rloop.get_best()+ data = {+ "attempts": [get_attempt_data(i) for i in range(self._i_attempt + 1)],+ **get_attempt_data(best_attempt_idx),+ }+ data["info"]["model_stats"] = self.attempt_model_stats.model_dump()+ data["extra_info"] = {"comparisons": [(a, b, comp.model_dump()) for a, b, comp in self._rloop.comparisons]}+ else:+ data = {+ **get_attempt_data(0),+ }assert self.traj_path is not Noneself.traj_path.write_text(json.dumps(data, indent=2))@@ -885,16 +907,16 @@ class Agent:# Run action/observation loopself._chook.on_run_start()- review_loop = get_review_loop_from_config(self.review_loop_config, problem_statement, self.model)step_output = StepOutput()while not step_output.done:step_output = self.step()self.save_trajectory()if step_output.done:- if review_loop is not None:- review_loop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))- self.info["review"] = review_loop.reviews[-1].to_dict()- if review_loop.retry():+ if self._rloop is not None:+ self._rloop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))+ self.info["review"] = self._rloop.reviews[-1].model_dump()+ self.save_trajectory()+ if self._rloop.retry():assert self._env is not Noneself.setup_attempt(hard_reset=True)step_output.done = Falsediff --git a/sweagent/agent/models.py b/sweagent/agent/models.pyindex f31142fc..da0730b8 100644--- a/sweagent/agent/models.py+++ b/sweagent/agent/models.py@@ -189,6 +189,11 @@ class InstanceStats(PydanticBaseModel):**{field: getattr(self, field) + getattr(other, field) for field in self.model_fields.keys()},)+ def __sub__(self, other: InstanceStats) -> InstanceStats:+ return InstanceStats(+ **{field: getattr(self, field) - getattr(other, field) for field in self.model_fields.keys()},+ )+class AbstractModel(ABC):def __init__(self, config: PydanticBaseModel, tools: ToolConfig):diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex 76544ed2..ffe91c52 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -89,7 +89,7 @@ class AbstractReviewLoop(ABC):@property@abstractmethoddef model_stats(self) -> InstanceStats:- """Returns the API stats of the model (if any)"""+ """Returns the API stats of the model"""@property@abstractmethod@@ -251,7 +251,7 @@ class Reviewer(AbstractReviewer):accept = self.interpret(answer)accept_emoji = "✅" if accept else "❌"logger.info(f"{self.LOG_PREFIX}{accept_emoji}\n{answer}")- return ReviewerResult(accept, answer, messages=messages)+ return ReviewerResult(accept=accept, output=answer, messages=messages)# todo: Couldn't I just replace the whole thing with Jinja templates?@@ -331,7 +331,7 @@ class BinaryReviewer(AbstractBinaryReviewer):"problem_statement": instance.get_problem_statement(),**instance.get_extra_fields(),}- user_message = self._config.instance_template.format(+ user_message = Template(self._config.instance_template).render(**ps_format_dict,**sub1.to_format_dict(suffix="1"),**sub2.to_format_dict(suffix="2"),@@ -368,7 +368,7 @@ class BinaryReviewer(AbstractBinaryReviewer):# Use words because else confusion with 0-based vs 1-based indiceschoice_emoji = "first" if idx == 0 else "second"logger.info(f"{self.LOG_PREFIX}{choice_emoji}\n{answer}")- return BinaryReviewerResult(idx, output=answer, messages=messages) # type: ignore+ return BinaryReviewerResult(choice=idx, output=answer, messages=messages) # type: ignoreclass GraveToCradle(AbstractGraveToCradle):@@ -391,7 +391,7 @@ class GraveToCradle(AbstractGraveToCradle):info = submissions[idx].infoif not info.get("submission"):continue- submission = info["submission"]+ submission = info["submission"] # type: ignorereview = reviews[idx].outputmsg_lines.append(f"Submission {i+1}:\n\n{submission}\n\nReview {i+1}:\n\n{review}")msg = "\n\n".join(msg_lines)diff --git a/sweagent/types.py b/sweagent/types.pyindex 3ac7f7dc..0a437695 100644--- a/sweagent/types.py+++ b/sweagent/types.py@@ -7,11 +7,9 @@ because of circular dependencies.from __future__ import annotationsimport copy-from dataclasses import dataclassfrom typing import Any, Literalfrom pydantic import BaseModel-from simple_parsing.helpers.serialization.serializable import FrozenSerializablefrom typing_extensions import TypedDict@@ -92,8 +90,7 @@ class AgentInfo(TypedDict, total=False):swe_rex_hash: str-@dataclass-class ReviewSubmission:+class ReviewSubmission(BaseModel):"""Information that's passed to the reviewer"""#: Total trajectory (including several retries)@@ -120,15 +117,13 @@ class ReviewSubmission:return out-@dataclass(frozen=True)-class ReviewerResult(FrozenSerializable):+class ReviewerResult(BaseModel):accept: booloutput: strmessages: list[dict[str, str]]-@dataclass(frozen=True)-class BinaryReviewerResult(FrozenSerializable):+class BinaryReviewerResult(BaseModel):choice: Literal[0, 1]output: strmessages: list[dict[str, str]]