π OpenHands: AI-Driven Development
Improve binary file handling and patch generation in SWE-bench evaluation (#7762)
Xingyao Wang008708264324
diff --git a/evaluation/benchmarks/swe_bench/binary_patch_utils.py b/evaluation/benchmarks/swe_bench/binary_patch_utils.pynew file mode 100644index 0000000000..9cf0dbd714--- /dev/null+++ b/evaluation/benchmarks/swe_bench/binary_patch_utils.py@@ -0,0 +1,52 @@+"""+Utilities for handling binary files and patch generation in SWE-bench evaluation.+"""+++def remove_binary_diffs(patch_text):+ """+ Remove binary file diffs from a git patch.++ Args:+ patch_text (str): The git patch text++ Returns:+ str: The cleaned patch text with binary diffs removed+ """+ lines = patch_text.splitlines()+ cleaned_lines = []+ block = []+ is_binary_block = False++ for line in lines:+ if line.startswith('diff --git '):+ if block and not is_binary_block:+ cleaned_lines.extend(block)+ block = [line]+ is_binary_block = False+ elif 'Binary files' in line:+ is_binary_block = True+ block.append(line)+ else:+ block.append(line)++ if block and not is_binary_block:+ cleaned_lines.extend(block)+ return '\n'.join(cleaned_lines)+++def remove_binary_files_from_git():+ """+ Generate a bash command to remove binary files from git staging.++ Returns:+ str: A bash command that removes binary files from git staging+ """+ return """+ for file in $(git status --porcelain | grep -E "^(M| M|\\?\\?|A| A)" | cut -c4-); do+ if [ -f "$file" ] && (file "$file" | grep -q "executable" || git check-attr binary "$file" | grep -q "binary: set"); then+ git rm -f "$file" 2>/dev/null || rm -f "$file"+ echo "Removed: $file"+ fi+ done+ """.strip()diff --git a/evaluation/benchmarks/swe_bench/run_infer.py b/evaluation/benchmarks/swe_bench/run_infer.pyindex 57ab8ed20d..49a0e4787d 100644--- a/evaluation/benchmarks/swe_bench/run_infer.py+++ b/evaluation/benchmarks/swe_bench/run_infer.py@@ -10,6 +10,10 @@ import tomlfrom datasets import load_datasetimport openhands.agenthub+from evaluation.benchmarks.swe_bench.binary_patch_utils import (+ remove_binary_diffs,+ remove_binary_files_from_git,+)from evaluation.benchmarks.swe_bench.resource.mapping import (get_instance_resource_factor,)@@ -38,8 +42,12 @@ from openhands.core.config import (from openhands.core.logger import openhands_logger as loggerfrom openhands.core.main import create_runtime, run_controllerfrom openhands.critic import AgentFinishedCritic-from openhands.events.action import CmdRunAction, MessageAction-from openhands.events.observation import CmdOutputObservation, ErrorObservation+from openhands.events.action import CmdRunAction, FileReadAction, MessageAction+from openhands.events.observation import (+ CmdOutputObservation,+ ErrorObservation,+ FileReadObservation,+)from openhands.events.serialization.event import event_from_dict, event_to_dictfrom openhands.runtime.base import Runtimefrom openhands.utils.async_utils import call_async_from_sync@@ -226,16 +234,17 @@ def initialize_runtime(workspace_dir_name = _get_swebench_workspace_dir_name(instance)obs: CmdOutputObservation- # Set instance id+ # Set instance id and git configurationaction = CmdRunAction(- command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc"""+ command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc && git config --global core.pager "" && git config --global diff.binary false""")action.set_hard_timeout(600)logger.info(action, extra={'msg_type': 'ACTION'})obs = runtime.run_action(action)logger.info(obs, extra={'msg_type': 'OBSERVATION'})assert_and_raise(- obs.exit_code == 0, f'Failed to export SWE_INSTANCE_ID: {str(obs)}'+ obs.exit_code == 0,+ f'Failed to export SWE_INSTANCE_ID and configure git: {str(obs)}',)action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """)@@ -452,11 +461,22 @@ def complete_runtime(f'Failed to git add -A: {str(obs)}',)+ # Remove binary files from git staging+ action = CmdRunAction(command=remove_binary_files_from_git())+ action.set_hard_timeout(600)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ assert_and_raise(+ isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,+ f'Failed to remove binary files: {str(obs)}',+ )+n_retries = 0git_patch = Nonewhile n_retries < 5:action = CmdRunAction(- command=f'git diff --no-color --cached {instance["base_commit"]}'+ command=f'git diff --no-color --cached {instance["base_commit"]} > patch.diff')action.set_hard_timeout(max(300 + 100 * n_retries, 600))logger.info(action, extra={'msg_type': 'ACTION'})@@ -465,8 +485,28 @@ def complete_runtime(n_retries += 1if isinstance(obs, CmdOutputObservation):if obs.exit_code == 0:- git_patch = obs.content.strip()- break+ # Read the patch file+ action = FileReadAction(path='patch.diff')+ action.set_hard_timeout(max(300 + 100 * n_retries, 600))+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ if isinstance(obs, FileReadObservation):+ git_patch = obs.content+ break+ elif isinstance(obs, ErrorObservation):+ # Fall back to cat "patch.diff" to get the patch+ assert 'File could not be decoded as utf-8' in obs.content+ action = CmdRunAction(command='cat patch.diff')+ action.set_hard_timeout(max(300 + 100 * n_retries, 600))+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert isinstance(obs, CmdOutputObservation) and obs.exit_code == 0+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ git_patch = obs.content+ break+ else:+ assert_and_raise(False, f'Unexpected observation type: {str(obs)}')else:logger.info('Failed to get git diff, retrying...')sleep_if_should_continue(10)@@ -478,6 +518,9 @@ def complete_runtime(assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')+ # Remove binary diffs from the patch+ git_patch = remove_binary_diffs(git_patch)+logger.info('-' * 30)logger.info('END Runtime Completion Fn')logger.info('-' * 30)@@ -802,7 +845,11 @@ if __name__ == '__main__':with open(cur_output_file, 'r') as f:for line in f:instance = json.loads(line)- if instance['instance_id'] not in added_instance_ids:+ # Also make sure git_patch is not empty - otherwise we fall back to previous attempt (empty patch is worse than anything else)+ if (+ instance['instance_id'] not in added_instance_ids+ and instance['test_result']['git_patch'].strip()+ ):fout.write(line)added_instance_ids.add(instance['instance_id'])logger.info(