SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
First "ask colleagues" implementation
Kilian Lieretbedd434514db
diff --git a/config/function_calling_anthropic_openhands_ask_colleagues.yaml b/config/function_calling_anthropic_openhands_ask_colleagues.yamlnew file mode 100644index 00000000..ce3a2f9e--- /dev/null+++ b/config/function_calling_anthropic_openhands_ask_colleagues.yaml@@ -0,0 +1,48 @@+# This template is heavily inspired by anthropic and openhands+agent:+ templates:+ system_template: |-+ You are a helpful assistant that can interact with a computer to solve tasks.+ instance_template: |-+ <uploaded_files>+ {{working_dir}}+ </uploaded_files>+ I've uploaded a python code repository in the directory {{working_dir}}. Consider the following PR description:++ <pr_description>+ {{problem_statement}}+ </pr_description>++ Can you help me implement the necessary changes to the repository so that the requirements specified in the <pr_description> are met?+ I've already taken care of all changes to any of the test files described in the <pr_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!+ Your task is to make the minimal changes to non-tests files in the {{working_dir}} directory to ensure the <pr_description> is satisfied.+ Follow these steps to resolve the issue:+ 1. As a first step, it might be a good idea to find and read code relevant to the <pr_description>+ 2. Create a script to reproduce the error and execute it with `python <filename.py>` using the bash tool, to confirm the error+ 3. Edit the sourcecode of the repo to resolve the issue+ 4. Rerun your reproduce script and confirm that the error is fixed!+ 5. Think about edgecases and make sure your fix handles them as well+ Your thinking should be thorough and so it's fine if it's very long.+ next_step_template: |-+ OBSERVATION:+ {{observation}}++ Please propose a new action. You must include _ONE_ function/tool call.+ next_step_no_output_template: |-+ Your command ran successfully and did not produce any output.++ Please propose a new action. You must include _ONE_ function/tool call.+ demonstrations:+ - trajectories/demonstrations/str_replace_anthropic_demo.yaml+ put_demos_in_history: true+ tools:+ bundles:+ - path: tools/registry+ - path: tools/edit_anthropic+ - path: tools/submit+ enable_bash_tool: true+ parse_function:+ type: function_calling+ history_processors: []+ best_response_picker:+ type: ask_colleaguesdiff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.pyindex a271bfa5..21916540 100644--- a/sweagent/agent/agents.py+++ b/sweagent/agent/agents.py@@ -32,6 +32,7 @@ from sweagent.agent.reviewer import AbstractReviewLoop, ReviewLoopConfig, get_refrom sweagent.environment.swe_env import SWEEnvfrom sweagent.exceptions import ContextWindowExceededError, CostLimitExceededError, FormatErrorfrom sweagent.tools.parsing import (+ ActionOnlyParser,ThoughtActionParser,)from sweagent.tools.tools import ToolConfig, ToolHandler@@ -137,6 +138,13 @@ class AgentConfig(BaseModel):# pydantic configmodel_config = ConfigDict(extra="forbid")+ @model_validator(mode="after")+ def validate_n_samples(self) -> Self:+ if self.n_samples is not None and self.n_samples > 1 and self.best_response_picker is None:+ msg = "n_samples must be 1 when best_response_picker is not set"+ raise ValueError(msg)+ return self+class _BlockedActionError(Exception):"""Raised when the agent's action is blocked"""@@ -178,6 +186,10 @@ class Agent:self.model = modelself.templates = templatesself.tools = tools+ if isinstance(self.model, HumanThoughtModel):+ self.tools.config.parse_function = ThoughtActionParser()+ elif isinstance(self.model, HumanModel):+ self.tools.config.parse_function = ActionOnlyParser()self.history_processors = history_processorsself.max_requeries = max_requeriesself.logger = get_logger("swea-agent", emoji="🤠")@@ -209,6 +221,10 @@ class Agent:"""self._best_response_picker = best_response_picker++ if self._best_response_picker is not None:+ self._best_response_picker.setup(self.model, self.tools)+self._n_samples = n_samples@classmethod@@ -731,24 +747,18 @@ class Agent:# todo: Add all options to the extra infoif self._best_response_picker is not None:assert self._problem_statement is not None- parsed_actions = [self.tools.parse_actions(o) for o in output]- best_idx = self._best_response_picker.pick_best(+ best = self._best_response_picker.get_action(problem_statement=self._problem_statement,trajectory=self.trajectory,- thoughts=[o[0] for o in parsed_actions],- actions=[o[1] for o in parsed_actions],+ history=history,+ completions=output,)- output = output[best_idx]+ output = best.completion+ # todo: Handle history and trajectory+ step.extra_info.update(best.extra_info)step.output = output["message"]# todo: Can't I override the parser in __init__?- if isinstance(self.model, HumanThoughtModel):- # TODO: This might be a bit hacky- # consider changing sweagent/tools/tools.py:ToolConfig to enforce this.- step.thought, step.action = ThoughtActionParser()(output, self.tools.config.commands)- elif isinstance(self.model, HumanModel):- step.thought, step.action = "", output["message"]- else:- step.thought, step.action = self.tools.parse_actions(output)+ step.thought, step.action = self.tools.parse_actions(output)if output.get("tool_calls") is not None:step.tool_call_ids = [call["id"] for call in output["tool_calls"]]step.tool_calls = output["tool_calls"]@@ -899,6 +909,7 @@ class Agent:"execution_time": step.execution_time,"state": step.state,"messages": self.messages,+ "extra_info": step.extra_info,},)self.trajectory.append(trajectory_step)diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.pyindex 6a1b1ce8..f1a28c82 100644--- a/sweagent/agent/best_response_picker.py+++ b/sweagent/agent/best_response_picker.py@@ -1,30 +1,94 @@from abc import abstractmethod-from typing import Literal+from typing import Any, Literalfrom jinja2 import Templatefrom pydantic import BaseModelfrom sweagent.agent.models import AbstractModelfrom sweagent.agent.problem_statement import ProblemStatement+from sweagent.exceptions import FormatError+from sweagent.tools.tools import ToolHandlerfrom sweagent.types import Trajectoryfrom sweagent.utils.log import get_loggerlogger = get_logger(__name__)+class GetActionOutput(BaseModel):+ completion: dict[str, Any]+ messages: list[dict[str, Any]] = []+ trajectory_items: list[dict[str, Any]] = []+ extra_info: dict[str, Any] = {}++class AbstractBestActionPicker(BaseModel):+ def setup(self, model: AbstractModel, tools: ToolHandler):+ self._model = model+ self._tools = tools+@abstractmethod- def pick_best(self, problem_statement: ProblemStatement, trajectory: Trajectory, actions: list[str]) -> int:+ def get_action(+ self,+ problem_statement: ProblemStatement,+ trajectory: Trajectory,+ history: list[dict[str, Any]],+ completions: list[dict[str, Any]],+ ) -> GetActionOutput:+ """Returns action with tool calls"""pass+class AskColleagues(AbstractBestActionPicker):+ type: Literal["ask_colleagues"] = "ask_colleagues"++ def get_colleague_discussion(self, completions: list[dict[str, Any]]) -> str:+ """Concat all completions into a single string"""+ out = "Your colleagues had the following ideas: \n\n"+ n_parsed_ok = 0+ for i, completion in enumerate(completions):+ try:+ thought, action = self._tools.parse_actions(completion)+ except FormatError:+ logger.warning("Could not parse completion %s, skipping.", completion)+ continue+ n_parsed_ok += 1+ out += f"Thought (colleague {i}): {thought}\nProposed Action (colleague {i}): {action}\n\n"+ if n_parsed_ok == 0:+ msg = "No completions could be parsed."+ raise FormatError(msg)+ out += (+ "Please summarize and compare the ideas and propose and action to take. "+ "Finally choose one action to perform and explain it in detail and include it as a tool call. "+ "<important>You must include a thought and action (as a tool/function call). Do not try to invoke commands with triple backticks, use function calls instead.</important>"+ )+ return out++ def get_action(+ self,+ problem_statement: ProblemStatement,+ trajectory: Trajectory,+ history: list[dict[str, Any]],+ completions: list[dict[str, Any]],+ ) -> GetActionOutput:+ """Returns action with tool calls"""+ discussion = self.get_colleague_discussion(completions)+ logger.info(f"COLLEAGUE DISCUSSION:\n{discussion}")+ new_messages = [+ {"role": "user", "content": discussion},+ ]+ final_completion = self._model.query(history + new_messages) # type: ignore+ return GetActionOutput(+ completion=final_completion,+ extra_info={"colleagues": discussion},+ )++class BinaryTrajectoryComparison(AbstractBestActionPicker):+ type: Literal["binary_trajectory_comparison"] = "binary_trajectory_comparison"+system_template: strinstance_template: str- def __init__(self, model: AbstractModel):- self.model = model-def _format_trajectory(self, trajectory: Trajectory) -> str:steps = []for i, step in enumerate(trajectory):@@ -59,9 +123,17 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):{"role": "user", "content": user_message},]- def pick_best(- self, *, problem_statement: ProblemStatement, trajectory: Trajectory, thoughts: list[str], actions: list[str]- ) -> int:+ def get_action(+ self,+ *,+ problem_statement: ProblemStatement,+ trajectory: Trajectory,+ history: list[dict[str, Any]],+ completions: list[dict[str, Any]],+ ) -> GetActionOutput:+ parsed_completions = [self._tools.parse_actions(a) for a in completions]+ thoughts: list[str] = [h[0] for h in parsed_completions]+ actions: list[str] = [h[1] for h in parsed_completions]assert len(thoughts) == len(actions)best_idx = 0for i in range(len(actions)):@@ -73,11 +145,13 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):thought2=thoughts[i],action2=actions[i],)- response = self.model.query(messages)["message"] # type: ignore+ response = self._model.query(messages)["message"] # type: ignoreidx = self.interpret(response)best_idx = i if idx == 1 else best_idx- return best_idx+ return GetActionOutput(+ completion=completions[best_idx],+ )def interpret(self, response: str) -> Literal[0, 1]:"""Interpret response from LM. Note: 1-based indexing"""@@ -90,4 +164,4 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):return 0-BestResponsePicker = BinaryTrajectoryComparison+BestResponsePicker = BinaryTrajectoryComparison | AskColleaguesdiff --git a/sweagent/agent/models.py b/sweagent/agent/models.pyindex 375fb6b2..0f411f5a 100644--- a/sweagent/agent/models.py+++ b/sweagent/agent/models.py@@ -287,18 +287,24 @@ class HumanModel(AbstractModel):# Input has escaped things like \n, so we need to unescape itaction = action.encode("utf8").decode("unicode_escape")_handle_raise_commands(action)- return action # type: ignore+ return {"message": action}- def query(self, history: History, action_prompt: str = "> ") -> dict:+ def query(self, history: History, action_prompt: str = "> ", n: int | None = None) -> dict | list[dict]:"""Wrapper to separate action prompt from formatting"""- try:- return {"message": self._query(history, action_prompt)}- except KeyboardInterrupt:- print("^C (exit with ^D)")- return self.query(history, action_prompt)- except EOFError:- print("\nGoodbye!")- return {"message": "exit"}+ out = []+ n_samples = n or 1+ for _ in range(n_samples):+ try:+ out.append(self._query(history, action_prompt))+ except KeyboardInterrupt:+ print("^C (exit with ^D)")+ out.append(self.query(history, action_prompt))+ except EOFError:+ print("\nGoodbye!")+ out.append({"message": "exit"})+ if n is None:+ return out[0]+ return outclass HumanThoughtModel(HumanModel):diff --git a/sweagent/tools/parsing.py b/sweagent/tools/parsing.pyindex 4e28b1cf..11531e42 100644--- a/sweagent/tools/parsing.py+++ b/sweagent/tools/parsing.py@@ -83,6 +83,18 @@ class ActionParser(AbstractParseFunction, BaseModel):raise FormatError(msg)+class ActionOnlyParser(AbstractParseFunction, BaseModel):+ """Expects the model response to be a single command."""++ error_message: str = "No message found in model response."++ type: Literal["action_only"] = "action_only"+ """Type for (de)serialization. Do not change."""++ def __call__(self, model_response: dict, commands: list[Command], strict=False):+ return "", model_response["message"]++class ThoughtActionParser(AbstractParseFunction, BaseModel):"""Expects the model response to be a discussion followed by a command wrapped in backticks.@@ -408,6 +420,7 @@ class JsonParser(AbstractParseFunction, BaseModel):ParseFunction = (ActionParser| ThoughtActionParser+ | ActionOnlyParser| XMLThoughtActionParser| FunctionCallingParser| EditFormatdiff --git a/sweagent/types.py b/sweagent/types.pyindex 0a437695..4bae478a 100644--- a/sweagent/types.py+++ b/sweagent/types.py@@ -26,6 +26,7 @@ class StepOutput(BaseModel):tool_calls: list[dict[str, Any]] | None = Nonetool_call_ids: list[str] | None = None"""State of the environment at the end of the step"""+ extra_info: dict[str, Any] = {}def to_template_format_dict(self) -> dict[str, str | int | float | bool | None]:"""Used for formatting (error) prompt templates"""@@ -46,6 +47,7 @@ class TrajectoryStep(TypedDict):thought: strexecution_time: floatmessages: list[dict[str, Any]]+ extra_info: dict[str, Any]# required fields go here