Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Handle RnR cost limits

Kilian Lieretcabbc05e9d4b
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index 5a1de0a5..bb79997f 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -151,6 +151,7 @@ class RetryAgentConfig(BaseModel):
retry_loop: RetryLoopConfig
type: Literal["retry"] = "retry"
model_config = ConfigDict(extra="forbid")
+ total_cost_limit: float
AgentConfig = Annotated[DefaultAgentConfig | RetryAgentConfig, Field(discriminator="type")]
@@ -248,6 +249,16 @@ class RetryAgent(AbstractAgent):
def step(self) -> StepOutput:
assert self._agent is not None
+ current_total_cost = self._total_instance_stats.instance_cost + self._agent.model.stats.instance_cost
+ if current_total_cost > self.config.total_cost_limit:
+ # This is a bit of a hack to raise exit_cost within the sub-agent,
+ # which ensures we handle the cost limit properly within the sub-agent (autosubmission etc.)
+ self.logger.warning(
+ "Cost limit (all attempts) exceeded: %s > %s. Setting current agent's per-instance cost limit to 0.0",
+ current_total_cost,
+ self.config.total_cost_limit,
+ )
+ self._agent.model.config.per_instance_cost_limit = 0.0
try:
return self._agent.step()
except Exception as e:
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index aa7738ec..45ea10c5 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -23,7 +23,6 @@ from sweagent.agent.models import (
get_model,
)
from sweagent.agent.problem_statement import ProblemStatement
-from sweagent.exceptions import AttemptCostLimitExceededError
from sweagent.tools.tools import ToolConfig
from sweagent.types import ReviewerResult, ReviewSubmission, Trajectory, TrajectoryStep
from sweagent.utils.log import get_logger
@@ -124,8 +123,6 @@ class ScoreRetryLoopConfig(BaseModel):
#: If set > 0 and there are more than this number of consecutive attempts
#: with an 'exit cost' exit stats, the review loop will quit.
max_n_consec_exit_cost: int = 0
- #: Cost limit for attempt (<=0: no limit)
- attempt_cost_limit: float = 0.0
#: Minimal $ that need to be left in order for us to start a new attempt
min_budget_for_new_attempt: float = 0.0
#: Override model temperature for first len(list) attempts
@@ -324,11 +321,6 @@ class ScoreRetryLoop(AbstractRetryLoop):
self._submissions.append(submission)
self._review()
- def on_model_query(self, attempt_stats: InstanceStats):
- if 0 < self._loop_config.attempt_cost_limit <= attempt_stats.instance_cost:
- self.logger.info("Exiting retry loop: Cost limit exceeded")
- raise AttemptCostLimitExceededError()
-
def _review(self) -> float:
review = self._reviewer.review(self._problem_statement, self._submissions[-1])
self._reviews.append(review)
diff --git a/sweagent/exceptions.py b/sweagent/exceptions.py
index f430f574..a2b70ce6 100644
--- a/sweagent/exceptions.py
+++ b/sweagent/exceptions.py
@@ -40,9 +40,5 @@ class TotalCostLimitExceededError(CostLimitExceededError):
"""Raised when we exceed the total cost limit"""
-class AttemptCostLimitExceededError(CostLimitExceededError):
- """Raised when we exceed the cost limit for an attempt"""
-
-
class ContentPolicyViolationError(Exception):
"""Raised when the model response violates a content policy"""