Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Agent: Bug fixes for retry

Kilian Lieret00a6a1282027
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index 6a979e8a..03d675a9 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -28,7 +28,7 @@ from sweagent.agent.models import (
get_model,
)
from sweagent.agent.problem_statement import ProblemStatement, ProblemStatementConfig
-from sweagent.agent.reviewer import AbstractRetryLoop, ReviewLoopConfig, get_review_loop_from_config
+from sweagent.agent.reviewer import AbstractRetryLoop, RetryLoopConfig, get_retry_loop_from_config
from sweagent.environment.swe_env import SWEEnv
from sweagent.exceptions import (
ContentPolicyViolationError,
@@ -136,7 +136,7 @@ class AgentConfig(BaseModel):
"""Maximum number of times to requery the model after an error, such as a
formatting error, a blocked action, or a bash syntax error.
"""
- review_loop: ReviewLoopConfig | None = None
+ retry_loop: RetryLoopConfig | None = None
action_sampler: ActionSamplerConfig | None = None
# pydantic config
@@ -169,7 +169,7 @@ class Agent:
name: str = "main",
_catch_errors: bool = True,
_always_require_zero_exit_code: bool = False,
- review_loop_config: ReviewLoopConfig | None = None,
+ retry_loop_config: RetryLoopConfig | None = None,
action_sampler_config: ActionSamplerConfig | None = None,
):
"""The agent handles the behaviour of the model and how it interacts with the environment.
@@ -189,7 +189,7 @@ class Agent:
self.history_processors = history_processors
self.max_requeries = max_requeries
self.logger = get_logger("swea-agent", emoji="🤠")
- self.review_loop_config = review_loop_config
+ self.retry_loop_config = retry_loop_config
# Set in run method
self._env: SWEEnv | None = None
self._problem_statement: ProblemStatement | ProblemStatementConfig | None = None
@@ -233,7 +233,7 @@ class Agent:
history_processors=config.history_processors,
model=model,
max_requeries=config.max_requeries,
- review_loop_config=config.review_loop,
+ retry_loop_config=config.retry_loop,
action_sampler_config=config.action_sampler,
)
@@ -301,7 +301,7 @@ class Agent:
for processor in self.history_processors:
messages = processor(messages)
- return messages
+ return messages # type: ignore
@property
def attempt_model_stats(self) -> InstanceStats:
@@ -345,8 +345,8 @@ class Agent:
self._trajectory_by_attempt = defaultdict(list)
self._info_by_attempt = defaultdict(dict) # type: ignore
self._forwarded_vars = {}
- if self.review_loop_config is not None:
- self._rloop = get_review_loop_from_config(self.review_loop_config, problem_statement, self.model)
+ if self.retry_loop_config is not None:
+ self._rloop = get_retry_loop_from_config(self.retry_loop_config, problem_statement, self.model)
if self._rloop is not None:
self._forwarded_vars = self._rloop.get_forwarded_vars()
self._chook.on_tools_installation_started()
@@ -531,7 +531,7 @@ class Agent:
if self.templates.strategy_template is not None:
templates.append(self.templates.strategy_template)
- self._add_templated_messages_to_history(templates, **state)
+ self._add_templated_messages_to_history(templates, **state) # type: ignore
def get_trajectory_data(self) -> dict[str, Any]:
"""Get all data that we save in .traj files."""
@@ -563,7 +563,7 @@ class Agent:
data["info"]["best_attempt_idx"] = best_attempt_idx
# Total model stats
data["info"]["model_stats"] = self.model.stats.model_dump()
- data["extra_info"] = {"comparisons": [(a, b, comp.model_dump()) for a, b, comp in self._rloop.comparisons]}
+ # data["extra_info"] = {"comparisons": [(a, b, comp.model_dump()) for a, b, comp in self._rloop.comparisons]}
else:
data = {
**get_attempt_data(0),
@@ -1005,7 +1005,7 @@ class Agent:
if step_output.done:
if self._rloop is not None:
self._rloop.on_submit(ReviewSubmission(trajectory=self.trajectory, info=self.info))
- self.info["review"] = self._rloop.reviews[-1].model_dump()
+ self.info["review"] = self._rloop.reviews[-1].model_dump() # type: ignore
self.save_trajectory()
if self._rloop.retry():
assert self._env is not None
diff --git a/sweagent/agent/models.py b/sweagent/agent/models.py
index ec4b5d28..973e89d7 100644
--- a/sweagent/agent/models.py
+++ b/sweagent/agent/models.py
@@ -295,7 +295,6 @@ class HumanModel(AbstractModel):
self,
history: History,
action_prompt: str = "> ",
- **kwargs,
) -> dict:
"""Logic for handling user input to pass to SWEEnv"""
action = input(action_prompt)
@@ -329,7 +328,7 @@ class HumanModel(AbstractModel):
_handle_raise_commands(action)
return {"message": action}
- def query(self, history: History, action_prompt: str = "> ", n: int | None = None) -> dict | list[dict]:
+ def query(self, history: History, action_prompt: str = "> ", n: int | None = None, **kwargs) -> dict | list[dict]:
"""Wrapper to separate action prompt from formatting"""
out = []
n_samples = n or 1
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index d0b15bb8..cb9ade88 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -10,7 +10,7 @@ from abc import ABC, abstractmethod
from typing import Any, Literal
from jinja2 import Template
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
from sweagent.agent.models import AbstractModel, HumanModel, HumanThoughtModel, InstanceStats, LiteLLMModel
from sweagent.agent.problem_statement import ProblemStatement
@@ -116,6 +116,8 @@ class TrajFormatterConfig(BaseModel):
item_template: str = "Model: {{response}}\n\nObservation: {{observation}}"
only_show_last_n_output: int = 0
+ model_config = ConfigDict(extra="forbid")
+
class ReviewerConfig(BaseModel):
"""The configuration for the reviewer"""
@@ -130,6 +132,11 @@ class ReviewerConfig(BaseModel):
type: Literal["reviewer"] = "reviewer"
+ model_config = ConfigDict(extra="forbid")
+
+ def get_reviewer(self, model: AbstractModel) -> AbstractReviewer:
+ return Reviewer(self, model)
+
class BinaryReviewerConfig(BaseModel):
"""The configuration for the binary reviewer"""
@@ -140,10 +147,14 @@ class BinaryReviewerConfig(BaseModel):
type: Literal["binary_reviewer"] = "binary_reviewer"
+ model_config = ConfigDict(extra="forbid")
+
class GTCConfig(BaseModel):
"""The configuration for the GraveToCradle"""
+ model_config = ConfigDict(extra="forbid")
+
class ScoreRetryLoopConfig(BaseModel):
"""The configuration for the review loop"""
@@ -171,6 +182,8 @@ class ScoreRetryLoopConfig(BaseModel):
#: Override model temperature for first len(list) attempts
temperature_override: list[float] = [0.0]
+ model_config = ConfigDict(extra="forbid")
+
def validate(self):
"""Checks config. Raises `ValueError` in case of misconfiguration"""
if self.min_attempts < 1:
@@ -198,7 +211,7 @@ RetryLoopConfig = ScoreRetryLoopConfig
# --- IMPLEMENTATIONS ---
-class Reviewer:
+class Reviewer(AbstractReviewer):
def __init__(self, config: ReviewerConfig, model):
self._config = config
self._model = model
@@ -414,7 +427,7 @@ class ScoreRetryLoop(AbstractRetryLoop):
):
self._model = model
self._instance = instance
- self._reviewer: AbstractReviewer = globals()[loop_config.reviewer_classname](loop_config.reviewer_config, model) # type: ignore
+ self._reviewer: AbstractReviewer = loop_config.reviewer_config.get_reviewer(model)
self._loop_config = loop_config
# Note: These are "cumulative" submissions, i.e., they include all retries
# up to that point.
@@ -520,12 +533,14 @@ class ScoreRetryLoop(AbstractRetryLoop):
return True
def get_best(self) -> int:
+ if len(self._reviews) == 0:
+ return 0
best_score = max([r.accept for r in self._reviews])
best_indices = [i for i, r in enumerate(self._reviews) if abs(r.accept - best_score) <= 1e-10]
return best_indices[0]
-def get_review_loop_from_config(
+def get_retry_loop_from_config(
config: RetryLoopConfig | None, instance: ProblemStatement, model: AbstractModel
) -> AbstractRetryLoop | None:
if config is None: