π OpenHands: AI-Driven Development
Jetbrains CI Benchmark (#7811)
juanmichelini6bcebd4b9d02
diff --git a/evaluation/benchmarks/lca_ci_build_repair/.gitignore b/evaluation/benchmarks/lca_ci_build_repair/.gitignorenew file mode 100644index 0000000000..5b6b0720c9--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/.gitignore@@ -0,0 +1 @@+config.yamldiff --git a/evaluation/benchmarks/lca_ci_build_repair/README.MD b/evaluation/benchmarks/lca_ci_build_repair/README.MDnew file mode 100644index 0000000000..63c2af2c30--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/README.MD@@ -0,0 +1,35 @@+# CI Builds Repair Benchmark Integration++This module integrates the CI Builds Repair benchmark developed by [JetBrains-Research](https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair/ci-builds-repair-benchmark).++For more information, refer to the [GitHub repository](https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair/ci-builds-repair-benchmark) and the associated [research paper](https://arxiv.org/abs/2406.11612).+See notice below for details++## Setup++Before running any scripts, make sure to configure the benchmark by setting up `config.yaml`.+This benchmark pushes to JetBrains' private GitHub repository. You will to request a `token_gh` provided by their team, to run this benchmark.++## Inference++To run inference with your model:++```bash+./evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh llm.yourmodel+```++## Evaluation++To evaluate the predictions:++```bash+./evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh predictions_path_containing_output+```++## Results+The benchmark contains 68 instances, we skip instances #126 and #145, and only run 66 instances due to dockerization errors.++Due to running in live GitHub machines, the benchmark is sensitive to the date it is run. Even the golden patches in the dataset might present failures due to updates.+For example, on 2025-04-09, running the benchmark against the golden patches gave 57/67 successes, with 1 job left in the waiting list.++On 2025-04-10, running the benchmark full with OH and no oracle, 37 succeeded. That is 54% of the complete set of 68 instances and 64% of the 57 that succeed with golden patches.diff --git a/evaluation/benchmarks/lca_ci_build_repair/config_template.yaml b/evaluation/benchmarks/lca_ci_build_repair/config_template.yamlnew file mode 100644index 0000000000..64ab751f97--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/config_template.yaml@@ -0,0 +1,11 @@+LCA_PATH: path #where to clone lca-ci rep+model_name: OpenHands+benchmark_owner: ICML-25-BenchName-builds-repair+token_gh: your_token+#for lca-ci-repo+repos_folder: /path/to/repos # here the cloned repos would be stored+out_folder: /out/folder # here the result files would be stored+data_cache_dir: /data/cache/dir/ # here the cached dataset would be stored+username_gh: username-gh # your GitHub username+# test_username: test_user # username that would be displayed in the benchmark. Optional. If ommitted, username_gh would be used+language: Python # dataset language (now only Python is available)diff --git a/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.pynew file mode 100644index 0000000000..1c67a29de1--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py@@ -0,0 +1,242 @@+"""Implements evaluation on JetBrains CI builds repair baselines++Please see https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair+and https://huggingface.co/datasets/JetBrains-Research/lca-ci-builds-repair++TODOs:+- Add more flags+"""++import json+import os+from pathlib import Path++import ruamel.yaml++from evaluation.utils.shared import (+ EvalMetadata,+ get_default_sandbox_config_for_eval,+ make_metadata,+)+from openhands.core.config import (+ AppConfig,+ LLMConfig,+ get_parser,+ load_app_config,+)+from openhands.core.logger import openhands_logger as logger+from openhands.core.main import create_runtime+from openhands.events.action import CmdRunAction+from openhands.events.observation import CmdOutputObservation+from openhands.runtime.base import Runtime+from openhands.utils.async_utils import call_async_from_sync+++def get_config(+ metadata: EvalMetadata,+) -> AppConfig:+ sandbox_config = get_default_sandbox_config_for_eval()+ sandbox_config.base_container_image = 'python:3.12-bookworm'+ config = AppConfig(+ default_agent=metadata.agent_class,+ run_as_openhands=False,+ runtime='docker',+ max_iterations=metadata.max_iterations,+ sandbox=sandbox_config,+ # do not mount workspace+ workspace_base=None,+ workspace_mount_path=None,+ )+ config.set_llm_config(metadata.llm_config)+ agent_config = config.get_agent_config(metadata.agent_class)+ agent_config.enable_prompt_extensions = False+ return config+++config = load_app_config()+++def load_bench_config():+ script_dir = os.path.dirname(+ os.path.abspath(__file__)+ ) # Get the absolute path of the script+ config_path = os.path.join(script_dir, 'config.yaml')+ yaml = ruamel.yaml.YAML(typ='rt')+ with open(config_path, 'r') as file:+ return yaml.load(file)+++bench_config = load_bench_config()+++def run_eval(+ runtime: Runtime,+):+ """Run the evaluation and create report"""+ logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")+ obs: CmdOutputObservation++ lca_path = bench_config['LCA_PATH']+ lca_ci_path = os.path.join(+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'+ )++ model_name = bench_config['model_name']++ action = CmdRunAction(command=f'mkdir {lca_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command=f'cd {lca_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ lca_repo_url = 'https://github.com/juanmichelini/lca-baselines'+ action = CmdRunAction(command=f'git clone {lca_repo_url}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command=f'cd {lca_ci_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command='git switch open-hands-integration')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ script_dir = os.path.dirname(+ os.path.abspath(__file__)+ ) # Get the absolute path of the script+ config_path = os.path.join(script_dir, 'config.yaml')+ runtime.copy_to(config_path, lca_ci_path)++ token_gh = bench_config['token_gh']+ commandf = f'export TOKEN_GH={token_gh}'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ action = CmdRunAction(command='poetry install')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ # Set up the task environment+ commandf = f'poetry run python run_eval_jobs.py --model-name "{model_name}" --config-path "{lca_ci_path}/config.yaml" --job-ids-file "/tmp/output_lca.jsonl" --result-filename "testfile.jsonl" > /tmp/single_output.txt'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ logger.info(f'run_eval_jobs.py gave {obs.content} !')+ # assert obs.exit_code == 0++ commandf = 'cat /tmp/single_output.txt'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ logger.info(f' {commandf} gave {obs.content}!')++ testfile_path = os.path.join(bench_config['out_folder'], 'testfile.jsonl')+ commandf = f'cat {testfile_path}'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ report_str = obs.content++ logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")+ return report_str+++def process_predictions(predictions_path: str):+ output_path = Path(predictions_path)+ if output_path.suffix != '.jsonl':+ raise ValueError('output_path must end in .jsonl')++ output_lca_path = output_path.with_name(output_path.stem + '_lca.jsonl')++ with output_path.open() as infile, output_lca_path.open('w') as outfile:+ for line in infile:+ data = json.loads(line)+ json.dump(data.get('test_result'), outfile)+ outfile.write('\n')++ return str(output_lca_path)+++if __name__ == '__main__':+ parser = get_parser()+ parser.add_argument(+ '-s',+ '--eval-split',+ type=str,+ default='test',+ choices=['test'],+ help='data split to evaluate on, must be test',+ )+ parser.add_argument(+ '--predictions-path',+ type=str,+ help='Path to the directory containing the output.jsonl with the predictions.',+ )+ args, _ = parser.parse_known_args()++ data_split = args.eval_split++ llm_config = LLMConfig(model='dummy_model')++ metadata = make_metadata(+ llm_config,+ f'jetbrains-lca-ci--{data_split}',+ args.agent_cls,+ args.max_iterations,+ args.eval_note,+ args.predictions_path,+ )++ # prepare image+ config = get_config(metadata)+ runtime = create_runtime(config)+ call_async_from_sync(runtime.connect)+ logger.info('Converting output.jsonl into output_lca.jsonl')+ predictions_lca_path = process_predictions(+ os.path.join(args.predictions_path, 'output.jsonl')+ )+ runtime.copy_to(predictions_lca_path, '/tmp')++ # get results+ results_str = run_eval(runtime)+ results_path = os.path.join(args.predictions_path, 'results.jsonl')+ with open(results_path, 'w') as file:+ file.write(results_str)+ logger.info(f'Saved results to {results_path}')++ # make a summary+ resolved_instances = []+ unresolved_instances = []+ for line in results_str.strip().splitlines():+ data = json.loads(line)+ conclusion = data.get('conclusion')+ if conclusion == 'success':+ resolved_instances.append(data)+ elif conclusion == 'failure':+ unresolved_instances.append(data)++ completed_instances = resolved_instances + unresolved_instances++ report = {+ 'success': len(resolved_instances),+ 'failure': len(unresolved_instances),+ 'resolved_instances': resolved_instances,+ 'unresolved_instances': unresolved_instances,+ 'completed_instances': completed_instances,+ }++ print(f'Results: {report}')+ report_path = os.path.join(args.predictions_path, 'report.jsonl')+ with open(report_path, 'w') as out_f:+ out_f.write(json.dumps(report) + '\n')++ logger.info(f'Saved report of results in swebench format to {report_path}')diff --git a/evaluation/benchmarks/lca_ci_build_repair/run_infer.py b/evaluation/benchmarks/lca_ci_build_repair/run_infer.pynew file mode 100644index 0000000000..e88a3a41fd--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/run_infer.py@@ -0,0 +1,406 @@+"""Implements inference on JetBrains CI builds repair baselines++Please see https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair+and https://huggingface.co/datasets/JetBrains-Research/lca-ci-builds-repair++TODOs:+- Add EXP_NAME+"""++import asyncio+import json+import os+from typing import Any++import pandas as pd+import ruamel.yaml+from datasets import load_dataset++from evaluation.utils.shared import (+ EvalMetadata,+ EvalOutput,+ codeact_user_response,+ compatibility_for_eval_history_pairs,+ get_default_sandbox_config_for_eval,+ make_metadata,+ prepare_dataset,+ reset_logger_for_multiprocessing,+ run_evaluation,+)+from openhands.controller.state.state import State+from openhands.core.config import (+ AppConfig,+ get_llm_config_arg,+ get_parser,+ load_app_config,+)+from openhands.core.logger import openhands_logger as logger+from openhands.core.main import create_runtime, run_controller+from openhands.events.action import CmdRunAction, MessageAction+from openhands.events.observation import CmdOutputObservation+from openhands.runtime.base import Runtime+from openhands.utils.async_utils import call_async_from_sync+++def get_config(+ metadata: EvalMetadata,+) -> AppConfig:+ sandbox_config = get_default_sandbox_config_for_eval()+ sandbox_config.base_container_image = 'python:3.12-bookworm'+ config = AppConfig(+ default_agent=metadata.agent_class,+ run_as_openhands=False,+ runtime='docker',+ max_iterations=metadata.max_iterations,+ sandbox=sandbox_config,+ # do not mount workspace+ workspace_base=None,+ workspace_mount_path=None,+ )+ config.set_llm_config(metadata.llm_config)+ agent_config = config.get_agent_config(metadata.agent_class)+ agent_config.enable_prompt_extensions = False+ return config+++config = load_app_config()+++def load_bench_config():+ script_dir = os.path.dirname(+ os.path.abspath(__file__)+ ) # Get the absolute path of the script+ config_path = os.path.join(script_dir, 'config.yaml')+ yaml = ruamel.yaml.YAML(typ='rt')+ with open(config_path, 'r') as file:+ return yaml.load(file)+++bench_config = load_bench_config()++AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {+ 'CodeActAgent': codeact_user_response,+}++AGENT_CLS_TO_INST_SUFFIX = {+ 'CodeActAgent': 'When you think you have completed the task, please finish the interaction using the "finish" tool.\n'+}+++def initialize_runtime(+ runtime: Runtime,+ instance: pd.Series,+):+ """Initialize the runtime for the agent.++ This function is called before the runtime is used to run the agent.+ """+ logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")+ obs: CmdOutputObservation++ lca_path = bench_config['LCA_PATH']+ lca_ci_path = os.path.join(+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'+ )++ repo_name = instance['repo_name']+ repos_path = bench_config['repos_folder']+ repo_owner = instance['repo_owner']+ repo_path = os.path.join(repos_path, f'{repo_owner}__{repo_name}')+ model_name = bench_config['model_name']++ action = CmdRunAction(command=f'mkdir {lca_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command=f'cd {lca_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ lca_repo_url = 'https://github.com/juanmichelini/lca-baselines'+ action = CmdRunAction(command=f'git clone {lca_repo_url}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command=f'cd {lca_ci_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ action = CmdRunAction(command='git switch open-hands-integration')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ script_dir = os.path.dirname(+ os.path.abspath(__file__)+ ) # Get the absolute path of the script+ config_path = os.path.join(script_dir, 'config.yaml')+ with open(config_path, 'r') as file:+ config_as_text = file.read()++ commandf = f"echo '{config_as_text}' > config.yaml"+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ token_gh = bench_config['token_gh']+ commandf = f'export TOKEN_GH={token_gh}'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ action = CmdRunAction(command='poetry install')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ # Set up the task environment+ commandf = f'poetry run python run_get_datapoint.py --model-name {model_name} --id {instance["id"]} > branch_name.txt'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ if obs.exit_code != 0:+ print(f'run_get_datapoint.py failed at {instance["id"]} with {obs.content}')+ assert obs.exit_code == 0++ commandf = 'cat branch_name.txt'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ bench_config['user_branch_name'] = obs.content++ # Navigate to the task's code path+ action = CmdRunAction(command=f'cd {repo_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")+++def complete_runtime(+ runtime: Runtime,+ instance: pd.Series,+) -> dict[str, Any]:+ """Complete the runtime for the agent.++ This function is called before the runtime is used to run the agent.+ If you need to do something in the sandbox to get the correctness metric after+ the agent has run, modify this function.+ """+ logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")+ obs: CmdOutputObservation++ model_name = bench_config['model_name']++ lca_path = bench_config['LCA_PATH']+ lca_ci_path = os.path.join(+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'+ )++ user_branch_name = bench_config['user_branch_name']++ token_gh = bench_config['token_gh']+ commandf = f'export TOKEN_GH={token_gh}'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)++ # Navigate to the lca-baseslines scripts path+ action = CmdRunAction(command=f'cd {lca_ci_path}')+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ assert obs.exit_code == 0++ commandf = f'poetry run python run_push_datapoint.py --id {instance["id"]} --model-name {model_name} --user-branch-name {user_branch_name} > single_output.json'+ logger.info(f'Running push script: {commandf}')+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ # assert obs.exit_code == 0++ commandf = 'cat single_output.json'+ action = CmdRunAction(command=commandf)+ logger.info(action, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action)+ result = json.loads(obs.content)++ logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")++ return result+++def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool = True):+ config = get_config(metadata)++ # Setup the logger properly, so you can run multi-processing to parallelize the evaluation+ if reset_logger:+ log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')+ reset_logger_for_multiprocessing(logger, instance['instance_id'], log_dir)+ else:+ logger.info(f'Starting evaluation for instance {instance["instance_id"]}.')++ repo_name = instance['repo_name']+ repo_workflow = instance['workflow_path']+ repo_logs = instance['logs']+ repos_path = bench_config['repos_folder']+ repo_owner = instance['repo_owner']+ repo_path = os.path.join(repos_path, f'{repo_owner}__{repo_name}')++ # Prepare the task instruction+ instruction_no_oracle = f"""+<uploaded_files>+{repo_path}+</uploaded_files>++I've uploaded a python code repository in the directory {repo_path}, Consider the following issue:++<issue_description>+The repository must pass the CI workflow {repo_workflow}.+but it gave the following error+{repo_logs}+</issue_description>++Can you help me implement the necessary changes to the repository so that the requirements specified in the <issue_description> are met?+I've already taken care of all changes to any of the test files described in the <issue_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!+Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages.+Your task is to make the minimal changes to non-test files in the {repo_path} directory to ensure the <issue_description> is satisfied.++Follow these phases to resolve the issue:++Phase 1. READING: read the problem and reword it in clearer terms+ 1.1 If there are code or config snippets. Express in words any best practices or conventions in them.+ 1.2 Hightlight message errors, method names, variables, file names, stack traces, and technical details.+ 1.3 Explain the problem in clear terms.+ 1.4 Enumerate the steps to reproduce the problem.+ 1.5 Hightlight any best practices to take into account when testing and fixing the issue++Phase 2. RUNNING: install and run the tests on the repository+ 2.1 Follow the readme+ 2.2 Install the environment and anything needed+ 2.2 Iterate and figure out how to run the tests++Phase 3. EXPLORATION: find the files that are related to the problem and possible solutions+ 3.1 Use `grep` to search for relevant methods, classes, keywords and error messages.+ 3.2 Identify all files related to the problem statement.+ 3.3 Propose the methods and files to fix the issue and explain why.+ 3.4 From the possible file locations, select the most likely location to fix the issue.++Phase 4. TEST CREATION: before implementing any fix, create a script to reproduce and verify the issue.+ 4.1 Look at existing test files in the repository to understand the test format/structure.+ 4.2 Create a minimal reproduction script that reproduces the located issue.+ 4.3 Run the reproduction script to confirm you are reproducing the issue.+ 4.4 Adjust the reproduction script as necessary.++Phase 5. FIX ANALYSIS: state clearly the problem and how to fix it+ 5.1 State clearly what the problem is.+ 5.2 State clearly where the problem is located.+ 5.3 State clearly how the test reproduces the issue.+ 5.4 State clearly the best practices to take into account in the fix.+ 5.5 State clearly how to fix the problem.++Phase 6. FIX IMPLEMENTATION: Edit the source code to implement your chosen solution.+ 6.1 Make minimal, focused changes to fix the issue.++Phase 7. VERIFICATION: Test your implementation thoroughly.+ 7.1 Run your reproduction script to verify the fix works.+ 7.2 Add edge cases to your test script to ensure comprehensive coverage.+ 7.3 Run existing tests related to the modified code to ensure you haven't broken anything. Run any tests in the repository related to:+ 7.2.1 The issue you are fixing+ 7.2.2 The files you modified+ 7.2.3 The functions you changed+ 7.4 If any tests fail, revise your implementation until all tests pass++Phase 8. REVIEW: Carefully re-read the problem description and compare your changes with the base commit {instance["sha_fail"]}.+ 8.1 Ensure you've fully addressed all requirements.++Once all phases are done, announce: 'Agent Task Complete'.+Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity.+"""+ runtime = create_runtime(config)+ call_async_from_sync(runtime.connect)+ initialize_runtime(runtime, instance)++ # Run the agent+ state: State | None = asyncio.run(+ run_controller(+ config=config,+ initial_user_action=MessageAction(content=instruction_no_oracle),+ runtime=runtime,+ fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(+ metadata.agent_class+ ),+ )+ )+ assert state is not None+ metrics = state.metrics.get() if state.metrics else {}++ test_result = complete_runtime(runtime, instance)++ # history is now available as a stream of events, rather than list of pairs of (Action, Observation)+ # for compatibility with the existing output format, we can remake the pairs here+ # remove when it becomes unnecessary+ histories = compatibility_for_eval_history_pairs(state.history)++ # Save the output+ output = EvalOutput(+ instance_id=instance['instance_id'],+ # instance=instance.to_dict(orient='recorods'),+ instruction=instruction_no_oracle,+ metadata=metadata,+ history=histories,+ test_result=test_result,+ metrics=metrics,+ )+ return output+++if __name__ == '__main__':+ parser = get_parser()+ parser.add_argument(+ '-s',+ '--eval-split',+ type=str,+ default='test',+ choices=['test'],+ help='data split to evaluate on, must be test',+ )+ args, _ = parser.parse_known_args()++ data_split = args.eval_split++ bench = load_dataset(+ 'JetBrains-Research/lca-ci-builds-repair', split=data_split+ ).to_pandas()+ # todo: see why 126 is giving problems on inference+ # todo: see why 145 is giving problems on eval+ bench = bench[bench['id'] != 126]+ bench = bench[bench['id'] != 145]+ # bench = bench.iloc[0:56]+ # add column instnace_id for compatibility with oh repo, old id column must be kept for lca repo+ bench['instance_id'] = bench['id'].astype(str)++ llm_config = None+ if args.llm_config:+ llm_config = get_llm_config_arg(args.llm_config)+ # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results+ llm_config.modify_params = False+ if llm_config is None:+ raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')++ metadata = make_metadata(+ llm_config,+ f'jetbrains-lca-ci--{data_split}',+ args.agent_cls,+ args.max_iterations,+ args.eval_note,+ args.eval_output_dir,+ )+ output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')+ instances = prepare_dataset(bench, output_file, args.eval_n_limit)++ run_evaluation(+ instances, metadata, output_file, args.eval_num_workers, process_instance+ )diff --git a/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh b/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.shnew file mode 100755index 0000000000..6c6a35c5ca--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh@@ -0,0 +1,33 @@+#!/usr/bin/env bash+set -eo pipefail++source "evaluation/utils/version_control.sh"++PROCESS_FILEPATH=$1+if [ -z "$PROCESS_FILEPATH" ]; then+ echo "Error: PROCESS_FILEPATH is empty. Usage: ./eval_infer.sh <output_file> [instance_id] [dataset_name] [split]"+ exit 1+fi++get_openhands_version++PROCESS_FILEPATH=$(realpath $PROCESS_FILEPATH)+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"+echo "PROCESS_FILEPATH: $PROCESS_FILEPATH"++EVAL_NOTE="$OPENHANDS_VERSION"+if [ -n "$EXP_NAME" ]; then+ EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"+fi++function run_eval() {+ COMMAND="poetry run python ./evaluation/benchmarks/lca_ci_build_repair/eval_infer.py \+ --predictions-path $PROCESS_FILEPATH "++ echo "RUNNING: $COMMAND"+ # Run the command+ eval $COMMAND+}++unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push+run_evaldiff --git a/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh b/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.shnew file mode 100755index 0000000000..c520e827f7--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh@@ -0,0 +1,27 @@+#!/usr/bin/env bash+set -eo pipefail++source "evaluation/utils/version_control.sh"++MODEL_CONFIG=$1++get_openhands_version++echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"+echo "MODEL_CONFIG: $MODEL_CONFIG"++EVAL_NOTE="$OPENHANDS_VERSION"+if [ -n "$EXP_NAME" ]; then+ EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"+fi++function run_eval() {+ COMMAND="poetry run python ./evaluation/benchmarks/lca_ci_build_repair/run_infer.py \+ --llm-config $MODEL_CONFIG "++ # Run the command+ eval $COMMAND+}++#unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push+run_evaldiff --git a/evaluation/benchmarks/lca_ci_build_repair/setup.py b/evaluation/benchmarks/lca_ci_build_repair/setup.pynew file mode 100644index 0000000000..1e02dd0a39--- /dev/null+++ b/evaluation/benchmarks/lca_ci_build_repair/setup.py@@ -0,0 +1,60 @@+"""Installs LCA CI Build Repair benchmark with scripts for OH integration."""++import os+import shutil+import subprocess++import yaml+++def setup():+ # Read config.yaml+ print('Reading config.yaml')+ script_dir = os.path.dirname(+ os.path.abspath(__file__)+ ) # Get the absolute path of the script+ config_path = os.path.join(script_dir, 'config.yaml')+ with open(config_path, 'r') as f:+ config = yaml.safe_load(f)++ lca_path = config['LCA_PATH']+ lca_ci_path = os.path.join(+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'+ )+ repo_url = 'https://github.com/juanmichelini/lca-baselines'++ # Clone the repository to LCA_CI_PATH+ print(f'Cloning lca-baselines repository from {repo_url} into {lca_path}')+ result = subprocess.run(+ ['git', 'clone', repo_url], cwd=lca_path, capture_output=True, text=True+ )+ if result.returncode != 0:+ print(f'Warning cloning repository: {result.stderr}')++ # Clone the repository to LCA_CI_PATH+ print('Switching branches')+ result = subprocess.run(+ ['git', 'switch', 'open-hands-integration'],+ cwd=lca_ci_path,+ capture_output=True,+ text=True,+ )+ if result.returncode != 0:+ print(f'Warning switching repository: {result.stderr}')++ # Move and rename config_lca.yaml (overwrite if exists)+ lca_ci_config_path = os.path.join(lca_ci_path, 'config.yaml')+ print(f'Copying config.yaml to {lca_ci_config_path}')+ shutil.copy(config_path, lca_ci_config_path)++ # Run poetry install in LCA_CI_PATH+ print(f"Running 'poetry install' in {lca_ci_path}")+ result = subprocess.run(+ ['poetry', 'install'], cwd=lca_ci_path, capture_output=True, text=True+ )+ if result.returncode != 0:+ print(f'Warning during poetry install: {result.stderr}')+++if __name__ == '__main__':+ setup()