SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Agent: Bug fixes for retry
Kilian Lieret00a6a1282027
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex 6a979e8a..03d675a9 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -28,7 +28,7 @@ from sweagent.agent.models import (get_model,)from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig-from sweagent.agent.reviewer import AbstractRetryLoop, ReviewLoopConfig, get_review_loop_from_config+from sweagent.agent.reviewer import AbstractRetryLoop, RetryLoopConfig, get_retry_loop_from_configfrom sweagent.environment.swe_env import SWEEnvfrom sweagent.exceptions import (ContentPolicyViolationError,@@ -136,7 +136,7 @@ class AgentConfig(BaseModel):"""Maximum number of times to requery the model after an error, such as aformatting error, a blocked action, or a bash syntax error."""- review_loop: ReviewLoopConfig | None = None+ retry_loop: RetryLoopConfig | None = Noneaction_sampler: ActionSamplerConfig | None = None# pydantic config@@ -169,7 +169,7 @@ class Agent:name: str = "main",_catch_errors: bool = True,_always_require_zero_exit_code: bool = False,- review_loop_config: ReviewLoopConfig | None = None,+ retry_loop_config: RetryLoopConfig | None = None,action_sampler_config: ActionSamplerConfig | None = None,):"""The agent handles the behaviour of the model and how it interacts with the environment.@@ -189,7 +189,7 @@ class Agent:self.history_processors = history_processorsself.max_requeries = max_requeriesself.logger = get_logger("swea-agent", emoji="🤠")- self.review_loop_config = review_loop_config+ self.retry_loop_config = retry_loop_config# Set in run methodself._env: SWEEnv | None = Noneself._problem_statement: ProblemStatement | ProblemStatementConfig | None = None@@ -233,7 +233,7 @@ class Agent:history_processors=config.history_processors,model=model,max_requeries=config.max_requeries,- review_loop_config=config.review_loop,+ retry_loop_config=config.retry_loop,action_sampler_config=config.action_sampler,)@@ -301,7 +301,7 @@ class Agent:for processor in self.history_processors:messages = processor(messages)- return messages+ return messages # type: ignore@propertydef attempt_model_stats(self) -> InstanceStats:@@ -345,8 +345,8 @@ class Agent:self._trajectory_by_attempt = defaultdict(list)self._info_by_attempt = defaultdict(dict) # type: ignoreself._forwarded_vars = {}- if self.review_loop_config is not None:- self._rloop = get_review_loop_from_config(self.review_loop_config, problem_statement, self.model)+ if self.retry_loop_config is not None:+ self._rloop = get_retry_loop_from_config(self.retry_loop_config, problem_statement, self.model)if self._rloop is not None:self._forwarded_vars = self._rloop.get_forwarded_vars()self._chook.on_tools_installation_started()@@ -531,7 +531,7 @@ class Agent:if self.templates.strategy_template is not None:templates.append(self.templates.strategy_template)- self._add_templated_messages_to_history(templates, **state)+ self._add_templated_messages_to_history(templates, **state) # type: ignoredef get_trajectory_data(self) -> dict[str, Any]:"""Get all data that we save in .traj files."""@@ -563,7 +563,7 @@ class Agent:data["info"]["best_attempt_idx"] = best_attempt_idx# Total model statsdata["info"]["model_stats"] = self.model.stats.model_dump()- data["extra_info"] = {"comparisons": [(a, b, comp.model_dump()) for a, b, comp in self._rloop.comparisons]}+ # data["extra_info"] = {"comparisons": [(a, b, comp.model_dump()) for a, b, comp in self._rloop.comparisons]}else:data = {**get_attempt_data(0),@@ -1005,7 +1005,7 @@ class Agent:if step_output.done:if self._rloop is not None:self._rloop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))- self.info["review"] = self._rloop.reviews[-1].model_dump()+ self.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignoreself.save_trajectory()if self._rloop.retry():assert self._env is not Nonediff --git a/sweagent/agent/models.py b/sweagent/agent/models.pyindex ec4b5d28..973e89d7 100644--- a/sweagent/agent/models.py+++ b/sweagent/agent/models.py@@ -295,7 +295,6 @@ class HumanModel(AbstractModel):self,history: History,action_prompt: str = "> ",- **kwargs,) -> dict:"""Logic for handling user input to pass to SWEEnv"""action = input(action_prompt)@@ -329,7 +328,7 @@ class HumanModel(AbstractModel):_handle_raise_commands(action)return {"message": action}- def query(self, history: History, action_prompt: str = "> ", n: int | None = None) -> dict | list[dict]:+ def query(self, history: History, action_prompt: str = "> ", n: int | None = None, **kwargs) -> dict | list[dict]:"""Wrapper to separate action prompt from formatting"""out = []n_samples = n or 1diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.pyindex d0b15bb8..cb9ade88 100644--- a/sweagent/agent/reviewer.py+++ b/sweagent/agent/reviewer.py@@ -10,7 +10,7 @@ from abc import ABC, abstractmethodfrom typing import Any, Literalfrom jinja2 import Template-from pydantic import BaseModel+from pydantic import BaseModel, ConfigDictfrom sweagent.agent.models import AbstractModel, HumanModel, HumanThoughtModel, InstanceStats, LiteLLMModelfrom sweagent.agent.problem_statement import ProblemStatement@@ -116,6 +116,8 @@ class TrajFormatterConfig(BaseModel):item_template: str = "Model: {{response}}\n\nObservation: {{observation}}"only_show_last_n_output: int = 0+ model_config = ConfigDict(extra="forbid")+class ReviewerConfig(BaseModel):"""The configuration for the reviewer"""@@ -130,6 +132,11 @@ class ReviewerConfig(BaseModel):type: Literal["reviewer"] = "reviewer"+ model_config = ConfigDict(extra="forbid")++ def get_reviewer(self, model: AbstractModel) -> AbstractReviewer:+ return Reviewer(self, model)+class BinaryReviewerConfig(BaseModel):"""The configuration for the binary reviewer"""@@ -140,10 +147,14 @@ class BinaryReviewerConfig(BaseModel):type: Literal["binary_reviewer"] = "binary_reviewer"+ model_config = ConfigDict(extra="forbid")+class GTCConfig(BaseModel):"""The configuration for the GraveToCradle"""+ model_config = ConfigDict(extra="forbid")+class ScoreRetryLoopConfig(BaseModel):"""The configuration for the review loop"""@@ -171,6 +182,8 @@ class ScoreRetryLoopConfig(BaseModel):#: Override model temperature for first len(list) attemptstemperature_override: list[float] = [0.0]+ model_config = ConfigDict(extra="forbid")+def validate(self):"""Checks config. Raises `ValueError` in case of misconfiguration"""if self.min_attempts < 1:@@ -198,7 +211,7 @@ RetryLoopConfig = ScoreRetryLoopConfig# --- IMPLEMENTATIONS ----class Reviewer:+class Reviewer(AbstractReviewer):def __init__(self, config: ReviewerConfig, model):self._config = configself._model = model@@ -414,7 +427,7 @@ class ScoreRetryLoop(AbstractRetryLoop):):self._model = modelself._instance = instance- self._reviewer: AbstractReviewer = globals()[loop_config.reviewer_classname](loop_config.reviewer_config, model) # type: ignore+ self._reviewer: AbstractReviewer = loop_config.reviewer_config.get_reviewer(model)self._loop_config = loop_config# Note: These are "cumulative" submissions, i.e., they include all retries# up to that point.@@ -520,12 +533,14 @@ class ScoreRetryLoop(AbstractRetryLoop):return Truedef get_best(self) -> int:+ if len(self._reviews) == 0:+ return 0best_score = max([r.accept for r in self._reviews])best_indices = [i for i, r in enumerate(self._reviews) if abs(r.accept - best_score) <= 1e-10]return best_indices[0]-def get_review_loop_from_config(+def get_retry_loop_from_config(config: RetryLoopConfig | None, instance: ProblemStatement, model: AbstractModel) -> AbstractRetryLoop | None:if config is None: