Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

First "ask colleagues" implementation

Kilian Lieretbedd434514db
diff --git a/config/function_calling_anthropic_openhands_ask_colleagues.yaml b/config/function_calling_anthropic_openhands_ask_colleagues.yaml
new file mode 100644
index 00000000..ce3a2f9e
--- /dev/null
+++ b/config/function_calling_anthropic_openhands_ask_colleagues.yaml
@@ -0,0 +1,48 @@
+# This template is heavily inspired by anthropic and openhands
+agent:
+ templates:
+ system_template: |-
+ You are a helpful assistant that can interact with a computer to solve tasks.
+ instance_template: |-
+ <uploaded_files>
+ {{working_dir}}
+ </uploaded_files>
+ I've uploaded a python code repository in the directory {{working_dir}}. Consider the following PR description:
+
+ <pr_description>
+ {{problem_statement}}
+ </pr_description>
+
+ Can you help me implement the necessary changes to the repository so that the requirements specified in the <pr_description> are met?
+ I've already taken care of all changes to any of the test files described in the <pr_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+ Your task is to make the minimal changes to non-tests files in the {{working_dir}} directory to ensure the <pr_description> is satisfied.
+ Follow these steps to resolve the issue:
+ 1. As a first step, it might be a good idea to find and read code relevant to the <pr_description>
+ 2. Create a script to reproduce the error and execute it with `python <filename.py>` using the bash tool, to confirm the error
+ 3. Edit the sourcecode of the repo to resolve the issue
+ 4. Rerun your reproduce script and confirm that the error is fixed!
+ 5. Think about edgecases and make sure your fix handles them as well
+ Your thinking should be thorough and so it's fine if it's very long.
+ next_step_template: |-
+ OBSERVATION:
+ {{observation}}
+
+ Please propose a new action. You must include _ONE_ function/tool call.
+ next_step_no_output_template: |-
+ Your command ran successfully and did not produce any output.
+
+ Please propose a new action. You must include _ONE_ function/tool call.
+ demonstrations:
+ - trajectories/demonstrations/str_replace_anthropic_demo.yaml
+ put_demos_in_history: true
+ tools:
+ bundles:
+ - path: tools/registry
+ - path: tools/edit_anthropic
+ - path: tools/submit
+ enable_bash_tool: true
+ parse_function:
+ type: function_calling
+ history_processors: []
+ best_response_picker:
+ type: ask_colleagues
diff --git a/sweagent/agent/agents.py b/sweagent/agent/agents.py
index a271bfa5..21916540 100644
--- a/sweagent/agent/agents.py
+++ b/sweagent/agent/agents.py
@@ -32,6 +32,7 @@ from sweagent.agent.reviewer import AbstractReviewLoop, ReviewLoopConfig, get_re
from sweagent.environment.swe_env import SWEEnv
from sweagent.exceptions import ContextWindowExceededError, CostLimitExceededError, FormatError
from sweagent.tools.parsing import (
+ ActionOnlyParser,
ThoughtActionParser,
)
from sweagent.tools.tools import ToolConfig, ToolHandler
@@ -137,6 +138,13 @@ class AgentConfig(BaseModel):
# pydantic config
model_config = ConfigDict(extra="forbid")
+ @model_validator(mode="after")
+ def validate_n_samples(self) -> Self:
+ if self.n_samples is not None and self.n_samples > 1 and self.best_response_picker is None:
+ msg = "n_samples must be 1 when best_response_picker is not set"
+ raise ValueError(msg)
+ return self
+
class _BlockedActionError(Exception):
"""Raised when the agent's action is blocked"""
@@ -178,6 +186,10 @@ class Agent:
self.model = model
self.templates = templates
self.tools = tools
+ if isinstance(self.model, HumanThoughtModel):
+ self.tools.config.parse_function = ThoughtActionParser()
+ elif isinstance(self.model, HumanModel):
+ self.tools.config.parse_function = ActionOnlyParser()
self.history_processors = history_processors
self.max_requeries = max_requeries
self.logger = get_logger("swea-agent", emoji="🤠")
@@ -209,6 +221,10 @@ class Agent:
"""
self._best_response_picker = best_response_picker
+
+ if self._best_response_picker is not None:
+ self._best_response_picker.setup(self.model, self.tools)
+
self._n_samples = n_samples
@classmethod
@@ -731,24 +747,18 @@ class Agent:
# todo: Add all options to the extra info
if self._best_response_picker is not None:
assert self._problem_statement is not None
- parsed_actions = [self.tools.parse_actions(o) for o in output]
- best_idx = self._best_response_picker.pick_best(
+ best = self._best_response_picker.get_action(
problem_statement=self._problem_statement,
trajectory=self.trajectory,
- thoughts=[o[0] for o in parsed_actions],
- actions=[o[1] for o in parsed_actions],
+ history=history,
+ completions=output,
)
- output = output[best_idx]
+ output = best.completion
+ # todo: Handle history and trajectory
+ step.extra_info.update(best.extra_info)
step.output = output["message"]
# todo: Can't I override the parser in __init__?
- if isinstance(self.model, HumanThoughtModel):
- # TODO: This might be a bit hacky
- # consider changing sweagent/tools/tools.py:ToolConfig to enforce this.
- step.thought, step.action = ThoughtActionParser()(output, self.tools.config.commands)
- elif isinstance(self.model, HumanModel):
- step.thought, step.action = "", output["message"]
- else:
- step.thought, step.action = self.tools.parse_actions(output)
+ step.thought, step.action = self.tools.parse_actions(output)
if output.get("tool_calls") is not None:
step.tool_call_ids = [call["id"] for call in output["tool_calls"]]
step.tool_calls = output["tool_calls"]
@@ -899,6 +909,7 @@ class Agent:
"execution_time": step.execution_time,
"state": step.state,
"messages": self.messages,
+ "extra_info": step.extra_info,
},
)
self.trajectory.append(trajectory_step)
diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.py
index 6a1b1ce8..f1a28c82 100644
--- a/sweagent/agent/best_response_picker.py
+++ b/sweagent/agent/best_response_picker.py
@@ -1,30 +1,94 @@
from abc import abstractmethod
-from typing import Literal
+from typing import Any, Literal
from jinja2 import Template
from pydantic import BaseModel
from sweagent.agent.models import AbstractModel
from sweagent.agent.problem_statement import ProblemStatement
+from sweagent.exceptions import FormatError
+from sweagent.tools.tools import ToolHandler
from sweagent.types import Trajectory
from sweagent.utils.log import get_logger
logger = get_logger(__name__)
+class GetActionOutput(BaseModel):
+ completion: dict[str, Any]
+ messages: list[dict[str, Any]] = []
+ trajectory_items: list[dict[str, Any]] = []
+ extra_info: dict[str, Any] = {}
+
+
class AbstractBestActionPicker(BaseModel):
+ def setup(self, model: AbstractModel, tools: ToolHandler):
+ self._model = model
+ self._tools = tools
+
@abstractmethod
- def pick_best(self, problem_statement: ProblemStatement, trajectory: Trajectory, actions: list[str]) -> int:
+ def get_action(
+ self,
+ problem_statement: ProblemStatement,
+ trajectory: Trajectory,
+ history: list[dict[str, Any]],
+ completions: list[dict[str, Any]],
+ ) -> GetActionOutput:
+ """Returns action with tool calls"""
pass
+class AskColleagues(AbstractBestActionPicker):
+ type: Literal["ask_colleagues"] = "ask_colleagues"
+
+ def get_colleague_discussion(self, completions: list[dict[str, Any]]) -> str:
+ """Concat all completions into a single string"""
+ out = "Your colleagues had the following ideas: \n\n"
+ n_parsed_ok = 0
+ for i, completion in enumerate(completions):
+ try:
+ thought, action = self._tools.parse_actions(completion)
+ except FormatError:
+ logger.warning("Could not parse completion %s, skipping.", completion)
+ continue
+ n_parsed_ok += 1
+ out += f"Thought (colleague {i}): {thought}\nProposed Action (colleague {i}): {action}\n\n"
+ if n_parsed_ok == 0:
+ msg = "No completions could be parsed."
+ raise FormatError(msg)
+ out += (
+ "Please summarize and compare the ideas and propose and action to take. "
+ "Finally choose one action to perform and explain it in detail and include it as a tool call. "
+ "<important>You must include a thought and action (as a tool/function call). Do not try to invoke commands with triple backticks, use function calls instead.</important>"
+ )
+ return out
+
+ def get_action(
+ self,
+ problem_statement: ProblemStatement,
+ trajectory: Trajectory,
+ history: list[dict[str, Any]],
+ completions: list[dict[str, Any]],
+ ) -> GetActionOutput:
+ """Returns action with tool calls"""
+ discussion = self.get_colleague_discussion(completions)
+ logger.info(f"COLLEAGUE DISCUSSION:\n{discussion}")
+ new_messages = [
+ {"role": "user", "content": discussion},
+ ]
+ final_completion = self._model.query(history + new_messages) # type: ignore
+ return GetActionOutput(
+ completion=final_completion,
+ extra_info={"colleagues": discussion},
+ )
+
+
class BinaryTrajectoryComparison(AbstractBestActionPicker):
+ type: Literal["binary_trajectory_comparison"] = "binary_trajectory_comparison"
+
system_template: str
instance_template: str
- def __init__(self, model: AbstractModel):
- self.model = model
-
def _format_trajectory(self, trajectory: Trajectory) -> str:
steps = []
for i, step in enumerate(trajectory):
@@ -59,9 +123,17 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
{"role": "user", "content": user_message},
]
- def pick_best(
- self, *, problem_statement: ProblemStatement, trajectory: Trajectory, thoughts: list[str], actions: list[str]
- ) -> int:
+ def get_action(
+ self,
+ *,
+ problem_statement: ProblemStatement,
+ trajectory: Trajectory,
+ history: list[dict[str, Any]],
+ completions: list[dict[str, Any]],
+ ) -> GetActionOutput:
+ parsed_completions = [self._tools.parse_actions(a) for a in completions]
+ thoughts: list[str] = [h[0] for h in parsed_completions]
+ actions: list[str] = [h[1] for h in parsed_completions]
assert len(thoughts) == len(actions)
best_idx = 0
for i in range(len(actions)):
@@ -73,11 +145,13 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
thought2=thoughts[i],
action2=actions[i],
)
- response = self.model.query(messages)["message"] # type: ignore
+ response = self._model.query(messages)["message"] # type: ignore
idx = self.interpret(response)
best_idx = i if idx == 1 else best_idx
- return best_idx
+ return GetActionOutput(
+ completion=completions[best_idx],
+ )
def interpret(self, response: str) -> Literal[0, 1]:
"""Interpret response from LM. Note: 1-based indexing"""
@@ -90,4 +164,4 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
return 0
-BestResponsePicker = BinaryTrajectoryComparison
+BestResponsePicker = BinaryTrajectoryComparison | AskColleagues
diff --git a/sweagent/agent/models.py b/sweagent/agent/models.py
index 375fb6b2..0f411f5a 100644
--- a/sweagent/agent/models.py
+++ b/sweagent/agent/models.py
@@ -287,18 +287,24 @@ class HumanModel(AbstractModel):
# Input has escaped things like \n, so we need to unescape it
action = action.encode("utf8").decode("unicode_escape")
_handle_raise_commands(action)
- return action # type: ignore
+ return {"message": action}
- def query(self, history: History, action_prompt: str = "> ") -> dict:
+ def query(self, history: History, action_prompt: str = "> ", n: int | None = None) -> dict | list[dict]:
"""Wrapper to separate action prompt from formatting"""
- try:
- return {"message": self._query(history, action_prompt)}
- except KeyboardInterrupt:
- print("^C (exit with ^D)")
- return self.query(history, action_prompt)
- except EOFError:
- print("\nGoodbye!")
- return {"message": "exit"}
+ out = []
+ n_samples = n or 1
+ for _ in range(n_samples):
+ try:
+ out.append(self._query(history, action_prompt))
+ except KeyboardInterrupt:
+ print("^C (exit with ^D)")
+ out.append(self.query(history, action_prompt))
+ except EOFError:
+ print("\nGoodbye!")
+ out.append({"message": "exit"})
+ if n is None:
+ return out[0]
+ return out
class HumanThoughtModel(HumanModel):
diff --git a/sweagent/tools/parsing.py b/sweagent/tools/parsing.py
index 4e28b1cf..11531e42 100644
--- a/sweagent/tools/parsing.py
+++ b/sweagent/tools/parsing.py
@@ -83,6 +83,18 @@ class ActionParser(AbstractParseFunction, BaseModel):
raise FormatError(msg)
+class ActionOnlyParser(AbstractParseFunction, BaseModel):
+ """Expects the model response to be a single command."""
+
+ error_message: str = "No message found in model response."
+
+ type: Literal["action_only"] = "action_only"
+ """Type for (de)serialization. Do not change."""
+
+ def __call__(self, model_response: dict, commands: list[Command], strict=False):
+ return "", model_response["message"]
+
+
class ThoughtActionParser(AbstractParseFunction, BaseModel):
"""
Expects the model response to be a discussion followed by a command wrapped in backticks.
@@ -408,6 +420,7 @@ class JsonParser(AbstractParseFunction, BaseModel):
ParseFunction = (
ActionParser
| ThoughtActionParser
+ | ActionOnlyParser
| XMLThoughtActionParser
| FunctionCallingParser
| EditFormat
diff --git a/sweagent/types.py b/sweagent/types.py
index 0a437695..4bae478a 100644
--- a/sweagent/types.py
+++ b/sweagent/types.py
@@ -26,6 +26,7 @@ class StepOutput(BaseModel):
tool_calls: list[dict[str, Any]] | None = None
tool_call_ids: list[str] | None = None
"""State of the environment at the end of the step"""
+ extra_info: dict[str, Any] = {}
def to_template_format_dict(self) -> dict[str, str | int | float | bool | None]:
"""Used for formatting (error) prompt templates"""
@@ -46,6 +47,7 @@ class TrajectoryStep(TypedDict):
thought: str
execution_time: float
messages: list[dict[str, Any]]
+ extra_info: dict[str, Any]
# required fields go here