Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Best response picker draft

Kilian Lieret8ec88f63993b
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index d780ecda..a271bfa5 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -16,6 +16,7 @@ from tenacity import RetryError
from typing_extensions import Self
from sweagent import __version__, get_agent_commit_hash, get_rex_commit_hash, get_rex_version
+from sweagent.agent.best_response_picker import BestResponsePicker
from sweagent.agent.history_processors import DefaultHistoryProcessor, HistoryProcessor
from sweagent.agent.hooks.abstract import AbstractAgentHook, CombinedAgentHook
from sweagent.agent.models import (
@@ -130,6 +131,8 @@ class AgentConfig(BaseModel):
formatting error, a blocked action, or a bash syntax error.
"""
review_loop: ReviewLoopConfig | None = None
+ best_response_picker: BestResponsePicker | None = None
+ n_samples: int = 1
# pydantic config
model_config = ConfigDict(extra="forbid")
@@ -162,6 +165,8 @@ class Agent:
_catch_errors: bool = True,
_always_require_zero_exit_code: bool = False,
review_loop_config: ReviewLoopConfig | None = None,
+ best_response_picker: BestResponsePicker | None = None,
+ n_samples: int = 1,
):
"""The agent handles the behaviour of the model and how it interacts with the environment.
@@ -203,6 +208,9 @@ class Agent:
It can be used to replay the agent's trajectory in an environment.
"""
+ self._best_response_picker = best_response_picker
+ self._n_samples = n_samples
+
@classmethod
def from_config(cls, config: AgentConfig) -> Self:
model = get_model(config.model, config.tools)
@@ -213,6 +221,8 @@ class Agent:
model=model,
max_requeries=config.max_requeries,
review_loop_config=config.review_loop,
+ best_response_picker=config.best_response_picker,
+ n_samples=config.n_samples,
)
def add_hook(self, hook: AbstractAgentHook) -> None:
@@ -717,8 +727,20 @@ class Agent:
if self._rloop is not None:
self._rloop.on_model_query(attempt_stats=self.attempt_model_stats)
self._chook.on_model_query(messages=history, agent=self.name)
- output = self.model.query(history) # type: ignore
+ output = self.model.query(history, n=self._n_samples if self._n_samples > 1 else None) # type: ignore
+ # todo: Add all options to the extra info
+ if self._best_response_picker is not None:
+ assert self._problem_statement is not None
+ parsed_actions = [self.tools.parse_actions(o) for o in output]
+ best_idx = self._best_response_picker.pick_best(
+ problem_statement=self._problem_statement,
+ trajectory=self.trajectory,
+ thoughts=[o[0] for o in parsed_actions],
+ actions=[o[1] for o in parsed_actions],
+ )
+ output = output[best_idx]
step.output = output["message"]
+ # todo: Can't I override the parser in __init__?
if isinstance(self.model, HumanThoughtModel):
# TODO: This might be a bit hacky
# consider changing sweagent/tools/tools.py:ToolConfig to enforce this.
diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.py
new file mode 100644
index 00000000..6a1b1ce8
--- /dev/null
+++ b/sweagent/agent/best_response_picker.py
@@ -0,0 +1,93 @@
+from abc import abstractmethod
+from typing import Literal
+
+from jinja2 import Template
+from pydantic import BaseModel
+
+from sweagent.agent.models import AbstractModel
+from sweagent.agent.problem_statement import ProblemStatement
+from sweagent.types import Trajectory
+from sweagent.utils.log import get_logger
+
+logger = get_logger(__name__)
+
+
+class AbstractBestActionPicker(BaseModel):
+ @abstractmethod
+ def pick_best(self, problem_statement: ProblemStatement, trajectory: Trajectory, actions: list[str]) -> int:
+ pass
+
+
+class BinaryTrajectoryComparison(AbstractBestActionPicker):
+ system_template: str
+ instance_template: str
+
+ def __init__(self, model: AbstractModel):
+ self.model = model
+
+ def _format_trajectory(self, trajectory: Trajectory) -> str:
+ steps = []
+ for i, step in enumerate(trajectory):
+ steps.append(f"Action {i}: {step['action']}\n Observation {i}: {step['observation']}")
+ return "\n".join(steps)
+
+ def format_messages(
+ self,
+ *,
+ problem_statement: ProblemStatement,
+ trajectory: Trajectory,
+ thought1: str,
+ action1: str,
+ thought2: str,
+ action2: str,
+ ) -> list[dict]:
+ system_message = self.system_template
+ logger.debug(f"MODEL INPUT (system)\n{system_message}")
+ ps_format_dict = {
+ "problem_statement": problem_statement.get_problem_statement(),
+ **problem_statement.get_extra_fields(),
+ }
+ user_message = Template(self.instance_template).render(
+ **ps_format_dict,
+ traj=self._format_trajectory(trajectory),
+ action1=action1,
+ action2=action2,
+ )
+ logger.debug(f"MODEL INPUT (user)\n{user_message}")
+ return [
+ {"role": "system", "content": system_message},
+ {"role": "user", "content": user_message},
+ ]
+
+ def pick_best(
+ self, *, problem_statement: ProblemStatement, trajectory: Trajectory, thoughts: list[str], actions: list[str]
+ ) -> int:
+ assert len(thoughts) == len(actions)
+ best_idx = 0
+ for i in range(len(actions)):
+ messages = self.format_messages(
+ problem_statement=problem_statement,
+ trajectory=trajectory,
+ thought1=thoughts[best_idx],
+ action1=actions[best_idx],
+ thought2=thoughts[i],
+ action2=actions[i],
+ )
+ response = self.model.query(messages)["message"] # type: ignore
+ idx = self.interpret(response)
+ best_idx = i if idx == 1 else best_idx
+
+ return best_idx
+
+ def interpret(self, response: str) -> Literal[0, 1]:
+ """Interpret response from LM. Note: 1-based indexing"""
+ last_line = response.strip().split("\n")[-1].strip()
+ if "first" in last_line.lower():
+ return 0
+ elif "second" in last_line.lower():
+ return 1
+ logger.warning("Could not interpret response: %s, will choose first submission.", response)
+ return 0
+
+
+BestResponsePicker = BinaryTrajectoryComparison
diff --git a/sweagent/agent/models.py b/sweagent/agent/models.py
index c3d7e525..375fb6b2 100644
--- a/sweagent/agent/models.py
+++ b/sweagent/agent/models.py
@@ -494,7 +494,7 @@ class LiteLLMModel(AbstractModel):
return None
return random.choice(api_keys)
- def _query(self, messages: list[dict[str, str]]) -> dict:
+ def _query(self, messages: list[dict[str, str]], n: int | None = None) -> dict | list[dict]:
input_tokens: int = litellm.utils.token_counter(messages=messages, model=self.config.name)
if self.model_max_input_tokens is None:
self.logger.warning(f"No max input tokens found for model {self.config.name!r}")
@@ -521,22 +521,30 @@ class LiteLLMModel(AbstractModel):
fallbacks=self.config.fallbacks,
**completion_kwargs,
**extra_args,
+ n=n,
)
- choices: litellm.types.utils.Choices = response.choices # type: ignore
- output = choices[0].message.content or ""
- output_dict = {"message": output}
cost = litellm.cost_calculator.completion_cost(response)
- output_tokens = litellm.utils.token_counter(text=output, model=self.config.name)
+ choices: litellm.types.utils.Choices = response.choices # type: ignore
+ n_choices = n if n is not None else 1
+ outputs = []
+ output_tokens = 0
+ for i in range(n_choices):
+ output = choices[i].message.content or ""
+ output_tokens += litellm.utils.token_counter(text=output, model=self.config.name)
+ output_dict = {"message": output}
+ if self.tools.use_function_calling:
+ if response.choices[i].message.tool_calls: # type: ignore
+ tool_calls = [call.to_dict() for call in response.choices[i].message.tool_calls] # type: ignore
+ else:
+ tool_calls = []
+ output_dict["tool_calls"] = tool_calls
+ outputs.append(output_dict)
self._update_stats(input_tokens=input_tokens, output_tokens=output_tokens, cost=cost)
- if self.tools.use_function_calling:
- if response.choices[0].message.tool_calls: # type: ignore
- tool_calls = [call.to_dict() for call in response.choices[0].message.tool_calls] # type: ignore
- else:
- tool_calls = []
- output_dict["tool_calls"] = tool_calls
- return output_dict
+ if n is None:
+ return outputs[0]
+ return outputs
- def query(self, history: History) -> dict:
+ def query(self, history: History, n: int | None = None) -> dict:
elapsed_time = time.time() - GLOBAL_STATS.last_query_timestamp
if elapsed_time < self.config.delay:
time.sleep(self.config.delay - elapsed_time)
@@ -571,7 +579,7 @@ class LiteLLMModel(AbstractModel):
f"{exception_info}"
)
messages = self._history_to_messages(history)
- result = self._query(messages)
+ result = self._query(messages, n=n)
return result
def _history_to_messages(
diff --git a/sweagent/agent/reviewer.py b/sweagent/agent/reviewer.py
index 3b80aab0..ef4666b6 100644
--- a/sweagent/agent/reviewer.py
+++ b/sweagent/agent/reviewer.py
@@ -40,11 +40,12 @@ class AbstractBinaryReviewer(ABC):
@abstractmethod
def compare_submissions(
self,
+ *,
instance: ProblemStatement,
sub1: ReviewSubmission,
sub2: ReviewSubmission,
- rev1: ReviewerResult,
- rev2: ReviewerResult,
+ rev1: ReviewerResult | None,
+ rev2: ReviewerResult | None,
) -> BinaryReviewerResult:
"""Returns 0 if sub1 is better, 1 if sub2 is better"""
@@ -363,8 +364,8 @@ class BinaryReviewer(AbstractBinaryReviewer):
instance: ProblemStatement,
sub1: ReviewSubmission,
sub2: ReviewSubmission,
- rev1: ReviewerResult,
- rev2: ReviewerResult,
+ rev1: ReviewerResult | None,
+ rev2: ReviewerResult | None,
) -> BinaryReviewerResult:
messages: History = self.format_messages(instance, sub1, sub2) # type: ignore
answer = self._model.query(messages)["message"]
@@ -495,7 +496,13 @@ class ReviewLoop(AbstractReviewLoop):
sub2 = self._submissions[-1]
rev1 = self._reviews[self._best_idx]
rev2 = self._reviews[-1]
- cresult = self._breviewer.compare_submissions(self._instance, sub1, sub2, rev1, rev2)
+ cresult = self._breviewer.compare_submissions(
+ instance=self._instance,
+ sub1=sub1,
+ sub2=sub2,
+ rev1=rev1,
+ rev2=rev2,
+ )
self._comparisons.append((self._n_samples - 2, self._n_samples - 1, cresult))
assert cresult.choice in [0, 1]
# this was a comparison between the current best and the last one