Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Fix(reviewer): Handle context window issues in reviewer

Kilian Liereta88c0871dda2
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index 3e17f2c4..8a6ed2a1 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -236,7 +236,11 @@ class Reviewer(AbstractReviewer):
answers = []
accepts = []
for _ in range(self._config.n_sample):
- answer = self._model.query(messages)["message"]
+ try:
+ answer = self._model.query(messages)["message"]
+ except Exception as e:
+ self.logger.warning(f"Query failed: {e}", exc_info=True)
+ continue
try:
score = self.interpret(answer)
except ValueError as e:
@@ -244,6 +248,9 @@ class Reviewer(AbstractReviewer):
continue
answers.append(answer)
accepts.append(score)
+ if not accepts:
+ answers = ["No valid scores found, failing submission"]
+ accepts = [-100.0]
accept = sum(accepts) / len(accepts) - penalty
std = np.std(accepts).item()
if self._config.reduce_by_std > 0: