π OpenHands: AI-Driven Development
feat(evaluation): Filter task ids by difficulty for SWE Gym rollouts (#11490)
Kevin Musgrave12d6da813050
diff --git a/evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py b/evaluation/benchmarks/multi_swe_bench/compute_skip_ids.pynew file mode 100644index 0000000000..1b2b6d36bb--- /dev/null+++ b/evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py@@ -0,0 +1,79 @@+import argparse+import fnmatch+import json+from collections import Counter+from pathlib import Path+++def find_final_reports(base_dir, pattern=None):+ base_path = Path(base_dir)+ if not base_path.exists():+ raise FileNotFoundError(f'Base directory does not exist: {base_dir}')++ # Find all final_report.json files+ all_reports = list(base_path.rglob('final_report.json'))++ if pattern is None:+ return all_reports++ # Filter by pattern+ filtered_reports = []+ for report in all_reports:+ # Get relative path from base_dir for matching+ rel_path = report.relative_to(base_path)+ if fnmatch.fnmatch(str(rel_path), pattern):+ filtered_reports.append(report)++ return filtered_reports+++def collect_resolved_ids(report_files):+ id_counter = Counter()++ for report_file in report_files:+ with open(report_file, 'r') as f:+ data = json.load(f)+ if 'resolved_ids' not in data:+ raise KeyError(f"'resolved_ids' key not found in {report_file}")+ resolved_ids = data['resolved_ids']+ id_counter.update(resolved_ids)++ return id_counter+++def get_skip_ids(id_counter, threshold):+ return [id_str for id_str, count in id_counter.items() if count >= threshold]+++def main():+ parser = argparse.ArgumentParser(+ description='Compute SKIP_IDS from resolved IDs in final_report.json files'+ )+ parser.add_argument(+ 'threshold',+ type=int,+ help='Minimum number of times an ID must be resolved to be skipped',+ )+ parser.add_argument(+ '--base-dir',+ default='evaluation/evaluation_outputs/outputs',+ help='Base directory to search for final_report.json files (default: evaluation/evaluation_outputs/outputs)',+ )+ parser.add_argument(+ '--pattern',+ default=None,+ help='Glob pattern to filter paths (e.g., "*Multi-SWE-RL*/**/*gpt*")',+ )++ args = parser.parse_args()+ report_files = find_final_reports(args.base_dir, args.pattern)+ id_counter = collect_resolved_ids(report_files)++ skip_ids = get_skip_ids(id_counter, args.threshold)+ skip_ids = [s.replace('/', '__').replace(':pr-', '-') for s in skip_ids]+ skip_ids = ','.join(sorted(skip_ids))+ print(skip_ids)+++if __name__ == '__main__':+ main()diff --git a/evaluation/benchmarks/multi_swe_bench/run_infer.py b/evaluation/benchmarks/multi_swe_bench/run_infer.pyindex d42879d7f8..333c235e90 100644--- a/evaluation/benchmarks/multi_swe_bench/run_infer.py+++ b/evaluation/benchmarks/multi_swe_bench/run_infer.py@@ -747,10 +747,14 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:subset = dataset[dataset[filter_column].isin(selected_ids)]logger.info(f'Retained {subset.shape[0]} tasks after filtering')return subset- skip_ids = os.environ.get('SKIP_IDS', '').split(',')+ skip_ids = [id for id in os.environ.get('SKIP_IDS', '').split(',') if id]if len(skip_ids) > 0:+ logger.info(f'Dataset size before filtering: {dataset.shape[0]} tasks')logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')- return dataset[~dataset[filter_column].isin(skip_ids)]+ logger.info(f'SKIP_IDS:\n{skip_ids}')+ filtered_dataset = dataset[~dataset[filter_column].isin(skip_ids)]+ logger.info(f'Dataset size after filtering: {filtered_dataset.shape[0]} tasks')+ return filtered_datasetreturn dataset@@ -768,6 +772,11 @@ if __name__ == '__main__':default='test',help='split to evaluate on',)+ parser.add_argument(+ '--filter_dataset_after_sampling',+ action='store_true',+ help='if provided, filter dataset after sampling instead of before',+ )args, _ = parser.parse_known_args()# NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing@@ -777,10 +786,24 @@ if __name__ == '__main__':logger.info(f'Loading dataset {args.dataset} with split {args.split} ')dataset = load_dataset('json', data_files=args.dataset)dataset = dataset[args.split]- swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')- logger.info(- f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'- )+ swe_bench_tests = dataset.to_pandas()++ # Determine filter strategy based on flag+ filter_func = None+ if args.filter_dataset_after_sampling:+ # Pass filter as callback to apply after sampling+ def filter_func(df):+ return filter_dataset(df, 'instance_id')++ logger.info(+ f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks (filtering will occur after sampling)'+ )+ else:+ # Apply filter before sampling+ swe_bench_tests = filter_dataset(swe_bench_tests, 'instance_id')+ logger.info(+ f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'+ )llm_config = Noneif args.llm_config:@@ -810,7 +833,9 @@ if __name__ == '__main__':output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')print(f'### OUTPUT FILE: {output_file} ###')- instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)+ instances = prepare_dataset(+ swe_bench_tests, output_file, args.eval_n_limit, filter_func=filter_func+ )if len(instances) > 0 and not isinstance(instances['FAIL_TO_PASS'][instances['FAIL_TO_PASS'].index[0]], strdiff --git a/evaluation/benchmarks/multi_swe_bench/scripts/rollout_multi_swegym.sh b/evaluation/benchmarks/multi_swe_bench/scripts/rollout_multi_swegym.shindex ed132432e3..cad572300b 100755--- a/evaluation/benchmarks/multi_swe_bench/scripts/rollout_multi_swegym.sh+++ b/evaluation/benchmarks/multi_swe_bench/scripts/rollout_multi_swegym.sh@@ -8,8 +8,14 @@MODEL=$1 # eg your llm config name in config.toml (eg: "llm.claude-3-5-sonnet-20241022-t05")EXP_NAME=$2 # "train-t05"EVAL_DATASET=$3 # path to original dataset (jsonl file)-N_WORKERS=${4:-64}-N_RUNS=${5:-1}+MAX_ITER=$4+N_WORKERS=${5:-64}+N_RUNS=${6:-1}+EVAL_LIMIT=${7:-}+SKIP_IDS_THRESHOLD=$8+SKIP_IDS_PATTERN=$9+INPUT_SKIP_IDS=${10}+FILTER_DATASET_AFTER_SAMPLING=${11:-}export EXP_NAME=$EXP_NAME# use 2x resources for rollout since some codebases are pretty resource-intensive@@ -17,6 +23,7 @@ export DEFAULT_RUNTIME_RESOURCE_FACTOR=2echo "MODEL: $MODEL"echo "EXP_NAME: $EXP_NAME"echo "EVAL_DATASET: $EVAL_DATASET"+echo "INPUT_SKIP_IDS: $INPUT_SKIP_IDS"# Generate DATASET path by adding _with_runtime_ before .jsonl extensionDATASET="${EVAL_DATASET%.jsonl}_with_runtime_.jsonl" # path to converted dataset@@ -35,9 +42,6 @@ elseexport SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"fi-#EVAL_LIMIT=3000-MAX_ITER=100-# ===== Run inference =====source "evaluation/utils/version_control.sh"@@ -69,17 +73,52 @@ function run_eval() {--dataset $DATASET \--split $SPLIT"+ # Conditionally add filter flag+ if [ "$FILTER_DATASET_AFTER_SAMPLING" = "true" ]; then+ COMMAND="$COMMAND --filter_dataset_after_sampling"+ fi+echo "Running command: $COMMAND"if [ -n "$EVAL_LIMIT" ]; thenecho "EVAL_LIMIT: $EVAL_LIMIT"COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"fi- # Run the commandeval $COMMAND}for run_idx in $(seq 1 $N_RUNS); do+ if [ -n "$SKIP_IDS_THRESHOLD" ]; then+ echo "Computing SKIP_IDS for run $run_idx..."+ SKIP_CMD="poetry run python evaluation/benchmarks/multi_swe_bench/compute_skip_ids.py $SKIP_IDS_THRESHOLD"+ if [ -n "$SKIP_IDS_PATTERN" ]; then+ SKIP_CMD="$SKIP_CMD --pattern \"$SKIP_IDS_PATTERN\""+ fi+ COMPUTED_SKIP_IDS=$(eval $SKIP_CMD)+ SKIP_STATUS=$?+ if [ $SKIP_STATUS -ne 0 ]; then+ echo "ERROR: Skip IDs computation failed with exit code $SKIP_STATUS"+ exit $SKIP_STATUS+ fi+ echo "COMPUTED_SKIP_IDS: $COMPUTED_SKIP_IDS"+ else+ echo "SKIP_IDS_THRESHOLD not provided, skipping SKIP_IDS computation"+ COMPUTED_SKIP_IDS=""+ fi++ # Concatenate COMPUTED_SKIP_IDS and INPUT_SKIP_IDS+ if [ -n "$COMPUTED_SKIP_IDS" ] && [ -n "$INPUT_SKIP_IDS" ]; then+ export SKIP_IDS="${COMPUTED_SKIP_IDS},${INPUT_SKIP_IDS}"+ elif [ -n "$COMPUTED_SKIP_IDS" ]; then+ export SKIP_IDS="$COMPUTED_SKIP_IDS"+ elif [ -n "$INPUT_SKIP_IDS" ]; then+ export SKIP_IDS="$INPUT_SKIP_IDS"+ else+ unset SKIP_IDS+ fi++ echo "FINAL SKIP_IDS: $SKIP_IDS"+ echo ""while true; doecho "### Running inference... ###"diff --git a/evaluation/utils/shared.py b/evaluation/utils/shared.pyindex a3d9c125af..287b0aa95e 100644--- a/evaluation/utils/shared.py+++ b/evaluation/utils/shared.py@@ -9,7 +9,7 @@ import timeimport tracebackfrom contextlib import contextmanagerfrom inspect import signature-from typing import Any, Awaitable, Callable, TextIO+from typing import Any, Awaitable, Callable, Optional, TextIOimport pandas as pdfrom pydantic import BaseModel@@ -222,6 +222,7 @@ def prepare_dataset(eval_n_limit: int,eval_ids: list[str] | None = None,skip_num: int | None = None,+ filter_func: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None,):assert 'instance_id' in dataset.columns, ("Expected 'instance_id' column in the dataset. You should define your own unique identifier for each instance and use it as the 'instance_id' column."@@ -265,6 +266,12 @@ def prepare_dataset(f'Randomly sampling {eval_n_limit} unique instances with random seed 42.')+ if filter_func is not None:+ dataset = filter_func(dataset)+ logger.info(+ f'Applied filter after sampling: {len(dataset)} instances remaining'+ )+def make_serializable(instance_dict: dict) -> dict:import numpy as np