SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Keep track of comparisons
Kilian Lieretb23ff4135479
diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.pyindex 5a5e59a8..7b269129 100644--- a/sweagent/agent/best_response_picker.py+++ b/sweagent/agent/best_response_picker.py@@ -178,7 +178,9 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):actions: list[str] = [h[1] for h in parsed_completions]assert len(thoughts) == len(actions)best_idx = 0+ comparison_log = []for i in range(1, len(actions)):+ # todo: Can't I just use the messages itself rather than turning them into a string?messages = self.format_messages(problem_statement=problem_statement,trajectory=trajectory,@@ -190,10 +192,19 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):response = self._model.query(messages)["message"] # type: ignorelogger.info(f"RESPONSE: {response}")idx = self.interpret(response)+ comparison_log.append(+ {+ "comparison_between": (best_idx, i),+ "messages": messages,+ "response": response,+ "idx": idx,+ }+ )best_idx = i if idx == 1 else best_idxreturn GetActionOutput(completion=completions[best_idx],+ extra_info={"comparison_log": comparison_log},)def interpret(self, response: str) -> Literal[0, 1]: