Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Feat(tools): Bring back linter as warning

Kilian Lieret1330d43559e8
diff --git a/tools/edit_anthropic/bin/str_replace_editor b/tools/edit_anthropic/bin/str_replace_editor
index 3ba9e6b3..3e7f9ad5 100644
--- a/tools/edit_anthropic/bin/str_replace_editor
+++ b/tools/edit_anthropic/bin/str_replace_editor
@@ -23,6 +23,28 @@ MAX_RESPONSE_LEN: int = 16000
MAX_WINDOW_EXPANSION_VIEW = int(REGISTRY.get("MAX_WINDOW_EXPANSION_VIEW", 0))
MAX_WINDOW_EXPANSION_EDIT_CONFIRM = int(REGISTRY.get("MAX_WINDOW_EXPANSION_EDIT_CONFIRM", 0))
USE_FILEMAP = REGISTRY.get("USE_FILEMAP", "false").lower() == "true"
+USE_LINTER = REGISTRY.get("USE_LINTER", "false").lower() == "true"
+Command = str
+SNIPPET_LINES: int = 4
+LINT_WARNING_TEMPLATE = """
+
+<NOTE>Your edits have been applied, but the linter has found syntax errors.</NOTE>
+
+<ERRORS>
+{errors}
+</ERRORS>
+
+Please review the changes and make sure they are correct.
+In addition to the above errors, please also check the following:
+
+1. The edited file is correctly indented
+2. The edited file does not contain duplicate lines
+3. The edit does not break existing functionality
+
+<IMPORTANT>In rare cases, the linter errors might not actually be errors or caused by your edit. Please use your own judgement.</IMPORTANT>
+
+Edit the file again if necessary.
+"""
def maybe_truncate(content: str, truncate_after: Optional[int] = MAX_RESPONSE_LEN):
@@ -34,8 +56,129 @@ def maybe_truncate(content: str, truncate_after: Optional[int] = MAX_RESPONSE_LE
)
-Command = str
-SNIPPET_LINES: int = 4
+class Flake8Error:
+ """A class to represent a single flake8 error"""
+
+ def __init__(self, filename: str, line_number: int, col_number: int, problem: str):
+ self.filename = filename
+ self.line_number = line_number
+ self.col_number = col_number
+ self.problem = problem
+
+ @classmethod
+ def from_line(cls, line: str):
+ try:
+ prefix, _sep, problem = line.partition(": ")
+ filename, line_number, col_number = prefix.split(":")
+ except (ValueError, IndexError) as e:
+ msg = f"Invalid flake8 error line: {line}"
+ raise ValueError(msg) from e
+ return cls(filename, int(line_number), int(col_number), problem)
+
+ def __eq__(self, other):
+ if not isinstance(other, Flake8Error):
+ return NotImplemented
+ return (
+ self.filename == other.filename
+ and self.line_number == other.line_number
+ and self.col_number == other.col_number
+ and self.problem == other.problem
+ )
+
+ def __repr__(self):
+ return f"Flake8Error(filename={self.filename}, line_number={self.line_number}, col_number={self.col_number}, problem={self.problem})"
+
+
+def _update_previous_errors(
+ previous_errors: List[Flake8Error], replacement_window: Tuple[int, int], replacement_n_lines: int
+) -> List[Flake8Error]:
+ """Update the line numbers of the previous errors to what they would be after the edit window.
+ This is a helper function for `_filter_previous_errors`.
+
+ All previous errors that are inside of the edit window should not be ignored,
+ so they are removed from the previous errors list.
+
+ Args:
+ previous_errors: list of errors with old line numbers
+ replacement_window: the window of the edit/lines that will be replaced
+ replacement_n_lines: the number of lines that will be used to replace the text
+
+ Returns:
+ list of errors with updated line numbers
+ """
+ updated = []
+ lines_added = replacement_n_lines - (replacement_window[1] - replacement_window[0] + 1)
+ for error in previous_errors:
+ if error.line_number < replacement_window[0]:
+ # no need to adjust the line number
+ updated.append(error)
+ continue
+ if replacement_window[0] <= error.line_number <= replacement_window[1]:
+ # The error is within the edit window, so let's not ignore it
+ # either way (we wouldn't know how to adjust the line number anyway)
+ continue
+ # We're out of the edit window, so we need to adjust the line number
+ updated.append(Flake8Error(error.filename, error.line_number + lines_added, error.col_number, error.problem))
+ return updated
+
+
+def format_flake8_output(
+ input_string: str,
+ show_line_numbers: bool = False,
+ *,
+ previous_errors_string: str = "",
+ replacement_window: Optional[Tuple[int, int]] = None,
+ replacement_n_lines: Optional[int] = None,
+) -> str:
+ """Filter flake8 output for previous errors and print it for a given file.
+
+ Args:
+ input_string: The flake8 output as a string
+ show_line_numbers: Whether to show line numbers in the output
+ previous_errors_string: The previous errors as a string
+ replacement_window: The window of the edit (lines that will be replaced)
+ replacement_n_lines: The number of lines used to replace the text
+
+ Returns:
+ The filtered flake8 output as a string
+ """
+ print(f"Replacement window: {replacement_window}")
+ print("Replacement n lines:", replacement_n_lines)
+ print("Previous errors string:", previous_errors_string)
+ print("Input string:", input_string)
+ errors = [Flake8Error.from_line(line.strip()) for line in input_string.split("\n") if line.strip()]
+ # print(f"New errors before filtering: {errors=}")
+ lines = []
+ if previous_errors_string:
+ assert replacement_window is not None
+ assert replacement_n_lines is not None
+ previous_errors = [
+ Flake8Error.from_line(line.strip()) for line in previous_errors_string.split("\n") if line.strip()
+ ]
+ print(f"Previous errors before updating: {previous_errors=}")
+ previous_errors = _update_previous_errors(previous_errors, replacement_window, replacement_n_lines)
+ print(f"Previous errors after updating: {previous_errors=}")
+ errors = [error for error in errors if error not in previous_errors]
+ # Sometimes new errors appear above the replacement window that were 'shadowed' by the previous errors
+ # they still clearly aren't caused by the edit.
+ errors = [error for error in errors if error.line_number >= replacement_window[0]]
+ print(f"New errors after filtering: {errors=}")
+ for error in errors:
+ if not show_line_numbers:
+ lines.append(f"- {error.problem}")
+ else:
+ lines.append(f"- line {error.line_number} col {error.col_number}: {error.problem}")
+ return "\n".join(lines)
+
+
+def flake8(file_path: str) -> str:
+ """Run flake8 on a given file and return the output as a string"""
+ if Path(file_path).suffix != ".py":
+ return ""
+ cmd = REGISTRY.get("LINT_COMMAND", "flake8 --isolated --select=F821,F822,F831,E111,E112,E113,E999,E902 {file_path}")
+ # don't use capture_output because it's not compatible with python3.6
+ out = subprocess.run(cmd.format(file_path=file_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ return out.stdout.decode()
class Filemap:
@@ -384,12 +527,42 @@ class EditTool:
print(f"No replacement was performed, old_str `{old_str}` is the same as new_str `{new_str}`.")
sys.exit(161)
+ pre_edit_lint = ""
+ if USE_LINTER:
+ try:
+ pre_edit_lint = flake8(str(path))
+ except Exception as e:
+ print(f"Warning: Failed to run pre-edit linter on {path}: {e}")
+
# Replace old_str with new_str
new_file_content = file_content.replace(old_str, new_str)
# Write the new content to the file
self.write_file(path, new_file_content)
+ post_edit_lint = ""
+ if USE_LINTER:
+ try:
+ post_edit_lint = flake8(str(path))
+ except Exception as e:
+ print(f"Warning: Failed to run post-edit linter on {path}: {e}")
+
+ epilogue = ""
+ if post_edit_lint:
+ ...
+ replacement_window_start_line = file_content.split(old_str)[0].count("\n") + 1
+ replacement_lines = len(new_str.split("\n"))
+ replacement_window_end_line = replacement_window_start_line + replacement_lines - 1
+ replacement_window = (replacement_window_start_line, replacement_window_end_line)
+ errors = format_flake8_output(
+ post_edit_lint,
+ previous_errors_string=pre_edit_lint,
+ replacement_window=replacement_window,
+ replacement_n_lines=replacement_lines,
+ )
+ if errors.strip():
+ epilogue = LINT_WARNING_TEMPLATE.format(errors=errors)
+
# Save the content to history
self._file_history[path].append(file_content)
@@ -406,6 +579,7 @@ class EditTool:
success_msg = f"The file {path} has been edited. "
success_msg += self._make_output(snippet, f"a snippet of {path}", start_line)
success_msg += "Review the changes and make sure they are as expected. Edit the file again if necessary."
+ success_msg += epilogue
print(success_msg)