Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Jetbrains CI Benchmark (#7811)

juanmichelini6bcebd4b9d02
diff --git a/evaluation/benchmarks/lca_ci_build_repair/.gitignore b/evaluation/benchmarks/lca_ci_build_repair/.gitignore
new file mode 100644
index 0000000000..5b6b0720c9
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/.gitignore
@@ -0,0 +1 @@
+config.yaml
diff --git a/evaluation/benchmarks/lca_ci_build_repair/README.MD b/evaluation/benchmarks/lca_ci_build_repair/README.MD
new file mode 100644
index 0000000000..63c2af2c30
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/README.MD
@@ -0,0 +1,35 @@
+# CI Builds Repair Benchmark Integration
+
+This module integrates the CI Builds Repair benchmark developed by [JetBrains-Research](https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair/ci-builds-repair-benchmark).
+
+For more information, refer to the [GitHub repository](https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair/ci-builds-repair-benchmark) and the associated [research paper](https://arxiv.org/abs/2406.11612).
+See notice below for details
+
+## Setup
+
+Before running any scripts, make sure to configure the benchmark by setting up `config.yaml`.
+This benchmark pushes to JetBrains' private GitHub repository. You will to request a `token_gh` provided by their team, to run this benchmark.
+
+## Inference
+
+To run inference with your model:
+
+```bash
+./evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh llm.yourmodel
+```
+
+## Evaluation
+
+To evaluate the predictions:
+
+```bash
+./evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh predictions_path_containing_output
+```
+
+## Results
+The benchmark contains 68 instances, we skip instances #126 and #145, and only run 66 instances due to dockerization errors.
+
+Due to running in live GitHub machines, the benchmark is sensitive to the date it is run. Even the golden patches in the dataset might present failures due to updates.
+For example, on 2025-04-09, running the benchmark against the golden patches gave 57/67 successes, with 1 job left in the waiting list.
+
+On 2025-04-10, running the benchmark full with OH and no oracle, 37 succeeded. That is 54% of the complete set of 68 instances and 64% of the 57 that succeed with golden patches.
diff --git a/evaluation/benchmarks/lca_ci_build_repair/config_template.yaml b/evaluation/benchmarks/lca_ci_build_repair/config_template.yaml
new file mode 100644
index 0000000000..64ab751f97
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/config_template.yaml
@@ -0,0 +1,11 @@
+LCA_PATH: path #where to clone lca-ci rep
+model_name: OpenHands
+benchmark_owner: ICML-25-BenchName-builds-repair
+token_gh: your_token
+#for lca-ci-repo
+repos_folder: /path/to/repos # here the cloned repos would be stored
+out_folder: /out/folder # here the result files would be stored
+data_cache_dir: /data/cache/dir/ # here the cached dataset would be stored
+username_gh: username-gh # your GitHub username
+# test_username: test_user # username that would be displayed in the benchmark. Optional. If ommitted, username_gh would be used
+language: Python # dataset language (now only Python is available)
diff --git a/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py
new file mode 100644
index 0000000000..1c67a29de1
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py
@@ -0,0 +1,242 @@
+"""Implements evaluation on JetBrains CI builds repair baselines
+
+Please see https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair
+and https://huggingface.co/datasets/JetBrains-Research/lca-ci-builds-repair
+
+TODOs:
+- Add more flags
+"""
+
+import json
+import os
+from pathlib import Path
+
+import ruamel.yaml
+
+from evaluation.utils.shared import (
+ EvalMetadata,
+ get_default_sandbox_config_for_eval,
+ make_metadata,
+)
+from openhands.core.config import (
+ AppConfig,
+ LLMConfig,
+ get_parser,
+ load_app_config,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime
+from openhands.events.action import CmdRunAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+
+def get_config(
+ metadata: EvalMetadata,
+) -> AppConfig:
+ sandbox_config = get_default_sandbox_config_for_eval()
+ sandbox_config.base_container_image = 'python:3.12-bookworm'
+ config = AppConfig(
+ default_agent=metadata.agent_class,
+ run_as_openhands=False,
+ runtime='docker',
+ max_iterations=metadata.max_iterations,
+ sandbox=sandbox_config,
+ # do not mount workspace
+ workspace_base=None,
+ workspace_mount_path=None,
+ )
+ config.set_llm_config(metadata.llm_config)
+ agent_config = config.get_agent_config(metadata.agent_class)
+ agent_config.enable_prompt_extensions = False
+ return config
+
+
+config = load_app_config()
+
+
+def load_bench_config():
+ script_dir = os.path.dirname(
+ os.path.abspath(__file__)
+ ) # Get the absolute path of the script
+ config_path = os.path.join(script_dir, 'config.yaml')
+ yaml = ruamel.yaml.YAML(typ='rt')
+ with open(config_path, 'r') as file:
+ return yaml.load(file)
+
+
+bench_config = load_bench_config()
+
+
+def run_eval(
+ runtime: Runtime,
+):
+ """Run the evaluation and create report"""
+ logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+ obs: CmdOutputObservation
+
+ lca_path = bench_config['LCA_PATH']
+ lca_ci_path = os.path.join(
+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'
+ )
+
+ model_name = bench_config['model_name']
+
+ action = CmdRunAction(command=f'mkdir {lca_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command=f'cd {lca_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ lca_repo_url = 'https://github.com/juanmichelini/lca-baselines'
+ action = CmdRunAction(command=f'git clone {lca_repo_url}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command=f'cd {lca_ci_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command='git switch open-hands-integration')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ script_dir = os.path.dirname(
+ os.path.abspath(__file__)
+ ) # Get the absolute path of the script
+ config_path = os.path.join(script_dir, 'config.yaml')
+ runtime.copy_to(config_path, lca_ci_path)
+
+ token_gh = bench_config['token_gh']
+ commandf = f'export TOKEN_GH={token_gh}'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ action = CmdRunAction(command='poetry install')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ # Set up the task environment
+ commandf = f'poetry run python run_eval_jobs.py --model-name "{model_name}" --config-path "{lca_ci_path}/config.yaml" --job-ids-file "/tmp/output_lca.jsonl" --result-filename "testfile.jsonl" > /tmp/single_output.txt'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ logger.info(f'run_eval_jobs.py gave {obs.content} !')
+ # assert obs.exit_code == 0
+
+ commandf = 'cat /tmp/single_output.txt'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ logger.info(f' {commandf} gave {obs.content}!')
+
+ testfile_path = os.path.join(bench_config['out_folder'], 'testfile.jsonl')
+ commandf = f'cat {testfile_path}'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ report_str = obs.content
+
+ logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+ return report_str
+
+
+def process_predictions(predictions_path: str):
+ output_path = Path(predictions_path)
+ if output_path.suffix != '.jsonl':
+ raise ValueError('output_path must end in .jsonl')
+
+ output_lca_path = output_path.with_name(output_path.stem + '_lca.jsonl')
+
+ with output_path.open() as infile, output_lca_path.open('w') as outfile:
+ for line in infile:
+ data = json.loads(line)
+ json.dump(data.get('test_result'), outfile)
+ outfile.write('\n')
+
+ return str(output_lca_path)
+
+
+if __name__ == '__main__':
+ parser = get_parser()
+ parser.add_argument(
+ '-s',
+ '--eval-split',
+ type=str,
+ default='test',
+ choices=['test'],
+ help='data split to evaluate on, must be test',
+ )
+ parser.add_argument(
+ '--predictions-path',
+ type=str,
+ help='Path to the directory containing the output.jsonl with the predictions.',
+ )
+ args, _ = parser.parse_known_args()
+
+ data_split = args.eval_split
+
+ llm_config = LLMConfig(model='dummy_model')
+
+ metadata = make_metadata(
+ llm_config,
+ f'jetbrains-lca-ci--{data_split}',
+ args.agent_cls,
+ args.max_iterations,
+ args.eval_note,
+ args.predictions_path,
+ )
+
+ # prepare image
+ config = get_config(metadata)
+ runtime = create_runtime(config)
+ call_async_from_sync(runtime.connect)
+ logger.info('Converting output.jsonl into output_lca.jsonl')
+ predictions_lca_path = process_predictions(
+ os.path.join(args.predictions_path, 'output.jsonl')
+ )
+ runtime.copy_to(predictions_lca_path, '/tmp')
+
+ # get results
+ results_str = run_eval(runtime)
+ results_path = os.path.join(args.predictions_path, 'results.jsonl')
+ with open(results_path, 'w') as file:
+ file.write(results_str)
+ logger.info(f'Saved results to {results_path}')
+
+ # make a summary
+ resolved_instances = []
+ unresolved_instances = []
+ for line in results_str.strip().splitlines():
+ data = json.loads(line)
+ conclusion = data.get('conclusion')
+ if conclusion == 'success':
+ resolved_instances.append(data)
+ elif conclusion == 'failure':
+ unresolved_instances.append(data)
+
+ completed_instances = resolved_instances + unresolved_instances
+
+ report = {
+ 'success': len(resolved_instances),
+ 'failure': len(unresolved_instances),
+ 'resolved_instances': resolved_instances,
+ 'unresolved_instances': unresolved_instances,
+ 'completed_instances': completed_instances,
+ }
+
+ print(f'Results: {report}')
+ report_path = os.path.join(args.predictions_path, 'report.jsonl')
+ with open(report_path, 'w') as out_f:
+ out_f.write(json.dumps(report) + '\n')
+
+ logger.info(f'Saved report of results in swebench format to {report_path}')
diff --git a/evaluation/benchmarks/lca_ci_build_repair/run_infer.py b/evaluation/benchmarks/lca_ci_build_repair/run_infer.py
new file mode 100644
index 0000000000..e88a3a41fd
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/run_infer.py
@@ -0,0 +1,406 @@
+"""Implements inference on JetBrains CI builds repair baselines
+
+Please see https://github.com/JetBrains-Research/lca-baselines/tree/main/ci-builds-repair
+and https://huggingface.co/datasets/JetBrains-Research/lca-ci-builds-repair
+
+TODOs:
+- Add EXP_NAME
+"""
+
+import asyncio
+import json
+import os
+from typing import Any
+
+import pandas as pd
+import ruamel.yaml
+from datasets import load_dataset
+
+from evaluation.utils.shared import (
+ EvalMetadata,
+ EvalOutput,
+ codeact_user_response,
+ compatibility_for_eval_history_pairs,
+ get_default_sandbox_config_for_eval,
+ make_metadata,
+ prepare_dataset,
+ reset_logger_for_multiprocessing,
+ run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+ AppConfig,
+ get_llm_config_arg,
+ get_parser,
+ load_app_config,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+
+def get_config(
+ metadata: EvalMetadata,
+) -> AppConfig:
+ sandbox_config = get_default_sandbox_config_for_eval()
+ sandbox_config.base_container_image = 'python:3.12-bookworm'
+ config = AppConfig(
+ default_agent=metadata.agent_class,
+ run_as_openhands=False,
+ runtime='docker',
+ max_iterations=metadata.max_iterations,
+ sandbox=sandbox_config,
+ # do not mount workspace
+ workspace_base=None,
+ workspace_mount_path=None,
+ )
+ config.set_llm_config(metadata.llm_config)
+ agent_config = config.get_agent_config(metadata.agent_class)
+ agent_config.enable_prompt_extensions = False
+ return config
+
+
+config = load_app_config()
+
+
+def load_bench_config():
+ script_dir = os.path.dirname(
+ os.path.abspath(__file__)
+ ) # Get the absolute path of the script
+ config_path = os.path.join(script_dir, 'config.yaml')
+ yaml = ruamel.yaml.YAML(typ='rt')
+ with open(config_path, 'r') as file:
+ return yaml.load(file)
+
+
+bench_config = load_bench_config()
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+ 'CodeActAgent': codeact_user_response,
+}
+
+AGENT_CLS_TO_INST_SUFFIX = {
+ 'CodeActAgent': 'When you think you have completed the task, please finish the interaction using the "finish" tool.\n'
+}
+
+
+def initialize_runtime(
+ runtime: Runtime,
+ instance: pd.Series,
+):
+ """Initialize the runtime for the agent.
+
+ This function is called before the runtime is used to run the agent.
+ """
+ logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+ obs: CmdOutputObservation
+
+ lca_path = bench_config['LCA_PATH']
+ lca_ci_path = os.path.join(
+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'
+ )
+
+ repo_name = instance['repo_name']
+ repos_path = bench_config['repos_folder']
+ repo_owner = instance['repo_owner']
+ repo_path = os.path.join(repos_path, f'{repo_owner}__{repo_name}')
+ model_name = bench_config['model_name']
+
+ action = CmdRunAction(command=f'mkdir {lca_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command=f'cd {lca_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ lca_repo_url = 'https://github.com/juanmichelini/lca-baselines'
+ action = CmdRunAction(command=f'git clone {lca_repo_url}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command=f'cd {lca_ci_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ action = CmdRunAction(command='git switch open-hands-integration')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ script_dir = os.path.dirname(
+ os.path.abspath(__file__)
+ ) # Get the absolute path of the script
+ config_path = os.path.join(script_dir, 'config.yaml')
+ with open(config_path, 'r') as file:
+ config_as_text = file.read()
+
+ commandf = f"echo '{config_as_text}' > config.yaml"
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ token_gh = bench_config['token_gh']
+ commandf = f'export TOKEN_GH={token_gh}'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ action = CmdRunAction(command='poetry install')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ # Set up the task environment
+ commandf = f'poetry run python run_get_datapoint.py --model-name {model_name} --id {instance["id"]} > branch_name.txt'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ if obs.exit_code != 0:
+ print(f'run_get_datapoint.py failed at {instance["id"]} with {obs.content}')
+ assert obs.exit_code == 0
+
+ commandf = 'cat branch_name.txt'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ bench_config['user_branch_name'] = obs.content
+
+ # Navigate to the task's code path
+ action = CmdRunAction(command=f'cd {repo_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+
+
+def complete_runtime(
+ runtime: Runtime,
+ instance: pd.Series,
+) -> dict[str, Any]:
+ """Complete the runtime for the agent.
+
+ This function is called before the runtime is used to run the agent.
+ If you need to do something in the sandbox to get the correctness metric after
+ the agent has run, modify this function.
+ """
+ logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+ obs: CmdOutputObservation
+
+ model_name = bench_config['model_name']
+
+ lca_path = bench_config['LCA_PATH']
+ lca_ci_path = os.path.join(
+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'
+ )
+
+ user_branch_name = bench_config['user_branch_name']
+
+ token_gh = bench_config['token_gh']
+ commandf = f'export TOKEN_GH={token_gh}'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+
+ # Navigate to the lca-baseslines scripts path
+ action = CmdRunAction(command=f'cd {lca_ci_path}')
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ assert obs.exit_code == 0
+
+ commandf = f'poetry run python run_push_datapoint.py --id {instance["id"]} --model-name {model_name} --user-branch-name {user_branch_name} > single_output.json'
+ logger.info(f'Running push script: {commandf}')
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ # assert obs.exit_code == 0
+
+ commandf = 'cat single_output.json'
+ action = CmdRunAction(command=commandf)
+ logger.info(action, extra={'msg_type': 'ACTION'})
+ obs = runtime.run_action(action)
+ result = json.loads(obs.content)
+
+ logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+
+ return result
+
+
+def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool = True):
+ config = get_config(metadata)
+
+ # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+ if reset_logger:
+ log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+ reset_logger_for_multiprocessing(logger, instance['instance_id'], log_dir)
+ else:
+ logger.info(f'Starting evaluation for instance {instance["instance_id"]}.')
+
+ repo_name = instance['repo_name']
+ repo_workflow = instance['workflow_path']
+ repo_logs = instance['logs']
+ repos_path = bench_config['repos_folder']
+ repo_owner = instance['repo_owner']
+ repo_path = os.path.join(repos_path, f'{repo_owner}__{repo_name}')
+
+ # Prepare the task instruction
+ instruction_no_oracle = f"""
+<uploaded_files>
+{repo_path}
+</uploaded_files>
+
+I've uploaded a python code repository in the directory {repo_path}, Consider the following issue:
+
+<issue_description>
+The repository must pass the CI workflow {repo_workflow}.
+but it gave the following error
+{repo_logs}
+</issue_description>
+
+Can you help me implement the necessary changes to the repository so that the requirements specified in the <issue_description> are met?
+I've already taken care of all changes to any of the test files described in the <issue_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages.
+Your task is to make the minimal changes to non-test files in the {repo_path} directory to ensure the <issue_description> is satisfied.
+
+Follow these phases to resolve the issue:
+
+Phase 1. READING: read the problem and reword it in clearer terms
+ 1.1 If there are code or config snippets. Express in words any best practices or conventions in them.
+ 1.2 Hightlight message errors, method names, variables, file names, stack traces, and technical details.
+ 1.3 Explain the problem in clear terms.
+ 1.4 Enumerate the steps to reproduce the problem.
+ 1.5 Hightlight any best practices to take into account when testing and fixing the issue
+
+Phase 2. RUNNING: install and run the tests on the repository
+ 2.1 Follow the readme
+ 2.2 Install the environment and anything needed
+ 2.2 Iterate and figure out how to run the tests
+
+Phase 3. EXPLORATION: find the files that are related to the problem and possible solutions
+ 3.1 Use `grep` to search for relevant methods, classes, keywords and error messages.
+ 3.2 Identify all files related to the problem statement.
+ 3.3 Propose the methods and files to fix the issue and explain why.
+ 3.4 From the possible file locations, select the most likely location to fix the issue.
+
+Phase 4. TEST CREATION: before implementing any fix, create a script to reproduce and verify the issue.
+ 4.1 Look at existing test files in the repository to understand the test format/structure.
+ 4.2 Create a minimal reproduction script that reproduces the located issue.
+ 4.3 Run the reproduction script to confirm you are reproducing the issue.
+ 4.4 Adjust the reproduction script as necessary.
+
+Phase 5. FIX ANALYSIS: state clearly the problem and how to fix it
+ 5.1 State clearly what the problem is.
+ 5.2 State clearly where the problem is located.
+ 5.3 State clearly how the test reproduces the issue.
+ 5.4 State clearly the best practices to take into account in the fix.
+ 5.5 State clearly how to fix the problem.
+
+Phase 6. FIX IMPLEMENTATION: Edit the source code to implement your chosen solution.
+ 6.1 Make minimal, focused changes to fix the issue.
+
+Phase 7. VERIFICATION: Test your implementation thoroughly.
+ 7.1 Run your reproduction script to verify the fix works.
+ 7.2 Add edge cases to your test script to ensure comprehensive coverage.
+ 7.3 Run existing tests related to the modified code to ensure you haven't broken anything. Run any tests in the repository related to:
+ 7.2.1 The issue you are fixing
+ 7.2.2 The files you modified
+ 7.2.3 The functions you changed
+ 7.4 If any tests fail, revise your implementation until all tests pass
+
+Phase 8. REVIEW: Carefully re-read the problem description and compare your changes with the base commit {instance["sha_fail"]}.
+ 8.1 Ensure you've fully addressed all requirements.
+
+Once all phases are done, announce: 'Agent Task Complete'.
+Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity.
+"""
+ runtime = create_runtime(config)
+ call_async_from_sync(runtime.connect)
+ initialize_runtime(runtime, instance)
+
+ # Run the agent
+ state: State | None = asyncio.run(
+ run_controller(
+ config=config,
+ initial_user_action=MessageAction(content=instruction_no_oracle),
+ runtime=runtime,
+ fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+ metadata.agent_class
+ ),
+ )
+ )
+ assert state is not None
+ metrics = state.metrics.get() if state.metrics else {}
+
+ test_result = complete_runtime(runtime, instance)
+
+ # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+ # for compatibility with the existing output format, we can remake the pairs here
+ # remove when it becomes unnecessary
+ histories = compatibility_for_eval_history_pairs(state.history)
+
+ # Save the output
+ output = EvalOutput(
+ instance_id=instance['instance_id'],
+ # instance=instance.to_dict(orient='recorods'),
+ instruction=instruction_no_oracle,
+ metadata=metadata,
+ history=histories,
+ test_result=test_result,
+ metrics=metrics,
+ )
+ return output
+
+
+if __name__ == '__main__':
+ parser = get_parser()
+ parser.add_argument(
+ '-s',
+ '--eval-split',
+ type=str,
+ default='test',
+ choices=['test'],
+ help='data split to evaluate on, must be test',
+ )
+ args, _ = parser.parse_known_args()
+
+ data_split = args.eval_split
+
+ bench = load_dataset(
+ 'JetBrains-Research/lca-ci-builds-repair', split=data_split
+ ).to_pandas()
+ # todo: see why 126 is giving problems on inference
+ # todo: see why 145 is giving problems on eval
+ bench = bench[bench['id'] != 126]
+ bench = bench[bench['id'] != 145]
+ # bench = bench.iloc[0:56]
+ # add column instnace_id for compatibility with oh repo, old id column must be kept for lca repo
+ bench['instance_id'] = bench['id'].astype(str)
+
+ llm_config = None
+ if args.llm_config:
+ llm_config = get_llm_config_arg(args.llm_config)
+ # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+ llm_config.modify_params = False
+ if llm_config is None:
+ raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+ metadata = make_metadata(
+ llm_config,
+ f'jetbrains-lca-ci--{data_split}',
+ args.agent_cls,
+ args.max_iterations,
+ args.eval_note,
+ args.eval_output_dir,
+ )
+ output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+ instances = prepare_dataset(bench, output_file, args.eval_n_limit)
+
+ run_evaluation(
+ instances, metadata, output_file, args.eval_num_workers, process_instance
+ )
diff --git a/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh b/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh
new file mode 100755
index 0000000000..6c6a35c5ca
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/scripts/eval_infer.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+PROCESS_FILEPATH=$1
+if [ -z "$PROCESS_FILEPATH" ]; then
+ echo "Error: PROCESS_FILEPATH is empty. Usage: ./eval_infer.sh <output_file> [instance_id] [dataset_name] [split]"
+ exit 1
+fi
+
+get_openhands_version
+
+PROCESS_FILEPATH=$(realpath $PROCESS_FILEPATH)
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "PROCESS_FILEPATH: $PROCESS_FILEPATH"
+
+EVAL_NOTE="$OPENHANDS_VERSION"
+if [ -n "$EXP_NAME" ]; then
+ EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+
+function run_eval() {
+ COMMAND="poetry run python ./evaluation/benchmarks/lca_ci_build_repair/eval_infer.py \
+ --predictions-path $PROCESS_FILEPATH "
+
+ echo "RUNNING: $COMMAND"
+ # Run the command
+ eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+run_eval
diff --git a/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh b/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh
new file mode 100755
index 0000000000..c520e827f7
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/scripts/run_infer.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+
+get_openhands_version
+
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+EVAL_NOTE="$OPENHANDS_VERSION"
+if [ -n "$EXP_NAME" ]; then
+ EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+
+function run_eval() {
+ COMMAND="poetry run python ./evaluation/benchmarks/lca_ci_build_repair/run_infer.py \
+ --llm-config $MODEL_CONFIG "
+
+ # Run the command
+ eval $COMMAND
+}
+
+#unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+run_eval
diff --git a/evaluation/benchmarks/lca_ci_build_repair/setup.py b/evaluation/benchmarks/lca_ci_build_repair/setup.py
new file mode 100644
index 0000000000..1e02dd0a39
--- /dev/null
+++ b/evaluation/benchmarks/lca_ci_build_repair/setup.py
@@ -0,0 +1,60 @@
+"""Installs LCA CI Build Repair benchmark with scripts for OH integration."""
+
+import os
+import shutil
+import subprocess
+
+import yaml
+
+
+def setup():
+ # Read config.yaml
+ print('Reading config.yaml')
+ script_dir = os.path.dirname(
+ os.path.abspath(__file__)
+ ) # Get the absolute path of the script
+ config_path = os.path.join(script_dir, 'config.yaml')
+ with open(config_path, 'r') as f:
+ config = yaml.safe_load(f)
+
+ lca_path = config['LCA_PATH']
+ lca_ci_path = os.path.join(
+ lca_path, 'lca-baselines', 'ci-builds-repair', 'ci-builds-repair-benchmark'
+ )
+ repo_url = 'https://github.com/juanmichelini/lca-baselines'
+
+ # Clone the repository to LCA_CI_PATH
+ print(f'Cloning lca-baselines repository from {repo_url} into {lca_path}')
+ result = subprocess.run(
+ ['git', 'clone', repo_url], cwd=lca_path, capture_output=True, text=True
+ )
+ if result.returncode != 0:
+ print(f'Warning cloning repository: {result.stderr}')
+
+ # Clone the repository to LCA_CI_PATH
+ print('Switching branches')
+ result = subprocess.run(
+ ['git', 'switch', 'open-hands-integration'],
+ cwd=lca_ci_path,
+ capture_output=True,
+ text=True,
+ )
+ if result.returncode != 0:
+ print(f'Warning switching repository: {result.stderr}')
+
+ # Move and rename config_lca.yaml (overwrite if exists)
+ lca_ci_config_path = os.path.join(lca_ci_path, 'config.yaml')
+ print(f'Copying config.yaml to {lca_ci_config_path}')
+ shutil.copy(config_path, lca_ci_config_path)
+
+ # Run poetry install in LCA_CI_PATH
+ print(f"Running 'poetry install' in {lca_ci_path}")
+ result = subprocess.run(
+ ['poetry', 'install'], cwd=lca_ci_path, capture_output=True, text=True
+ )
+ if result.returncode != 0:
+ print(f'Warning during poetry install: {result.stderr}')
+
+
+if __name__ == '__main__':
+ setup()