Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Enh(reviewer): Allow to set temperature separately

Kilian Lieretaf9fb1ae65dc
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index 72335f5d..ce704e1c 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -327,6 +327,7 @@ class Agent:
self.traj_path = output_dir / (self._problem_statement.id + ".traj")
self.logger.info("Trajectory will be saved to %s", self.traj_path)
+ # Gets inremented to 0 in setup_attempt which we call from here
self._i_attempt = -1
self._history_by_attempt = defaultdict(list)
self._trajectory_by_attempt = defaultdict(list)
@@ -346,6 +347,8 @@ class Agent:
"""Setup the agent for a new attempt."""
self._chook.on_setup_attempt()
self._i_attempt += 1
+ if self._rloop is not None:
+ self._rloop.on_attempt_started(self._i_attempt, self)
self.info = AgentInfo()
self.info["swe_agent_hash"] = get_agent_commit_hash()
self.info["swe_agent_version"] = __version__
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index f594f1c7..b3b1827d 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -11,7 +11,7 @@ from typing import Any, Literal
from jinja2 import Template
from pydantic import BaseModel
-from sweagent.agent.models import AbstractModel, HumanModel, HumanThoughtModel, InstanceStats
+from sweagent.agent.models import AbstractModel, HumanModel, HumanThoughtModel, InstanceStats, LiteLLMModel
from sweagent.agent.problem_statement import ProblemStatement
from sweagent.exceptions import AttemptCostLimitExceededError
from sweagent.types import BinaryReviewerResult, History, ReviewerResult, ReviewSubmission, Trajectory, TrajectoryStep
@@ -88,6 +88,10 @@ class AbstractReviewLoop(ABC):
stop conditions based on attempt cost etc.
"""
+ def on_attempt_started(self, i_attempt: int, agent):
+ """Called when a new attempt is started"""
+ pass
+
@abstractmethod
def get_best(self) -> int:
"""Returns the best solution"""
@@ -178,6 +182,8 @@ class ReviewLoopConfig(BaseModel):
attempt_cost_limit: float = 0.0
#: Minimal $ that need to be left in order for us to start a new attempt
min_budget_for_new_attempt: float = 0.0
+ #: Override model temperature for first len(list) attempts
+ temperature_override: list[float] = [0.0]
def validate(self):
"""Checks config. Raises `ValueError` in case of misconfiguration"""
@@ -436,6 +442,8 @@ class ReviewLoop(AbstractReviewLoop):
self._best_idx: int = 0
#: Number of consecutive exit cost submissions
self._n_consec_exit_cost: int = 0
+ #: Original temperature
+ self._terminal_temperature: float | None = None
# Properties
# ----------
@@ -458,6 +466,21 @@ class ReviewLoop(AbstractReviewLoop):
# -------
+ def on_attempt_started(self, i_attempt: int, agent: str) -> None:
+ if not isinstance(self._model, LiteLLMModel):
+ return
+ # Attempts are 1-indexed
+ logger.debug(f"{self.LOG_PREFIX}Setting temperature for attempt {i_attempt}")
+ if i_attempt == 0:
+ self._terminal_temperature = self._model.config.temperature
+ logger.debug(f"{self.LOG_PREFIX}Set terminal temperature to {self._terminal_temperature}")
+ if i_attempt < len(self._loop_config.temperature_override):
+ self._model.config.temperature = self._loop_config.temperature_override[i_attempt]
+ else:
+ assert self._terminal_temperature is not None
+ self._model.config.temperature = self._terminal_temperature
+ logger.debug(f"{self.LOG_PREFIX}Set temperature to {self._model.config.temperature}")
+
def on_submit(self, submission: ReviewSubmission) -> None:
self._submissions.append(submission)
self._review()