π OpenHands: AI-Driven Development
Evaluation: redirect sessions to repo-local .eval_sessions via helper; apply across entrypoints; add tests (#10540)
Xingyao Wang4507a25b854d
diff --git a/.gitignore b/.gitignoreindex f61f3ced52..f9fb4dc408 100644--- a/.gitignore+++ b/.gitignore@@ -257,3 +257,5 @@ containers/runtime/code# test resultstest-results++.eval_sessionsdiff --git a/evaluation/benchmarks/EDA/run_infer.py b/evaluation/benchmarks/EDA/run_infer.pyindex ff25172722..ed9c77ee7e 100644--- a/evaluation/benchmarks/EDA/run_infer.py+++ b/evaluation/benchmarks/EDA/run_infer.py@@ -9,8 +9,8 @@ from evaluation.utils.shared import (EvalMetadata,EvalOutput,compatibility_for_eval_history_pairs,- get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -61,18 +61,15 @@ AGENT_CLS_TO_INST_SUFFIX = {def get_config(metadata: EvalMetadata,) -> OpenHandsConfig:- sandbox_config = get_default_sandbox_config_for_eval()- sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ # Create config with EDA-specific container image+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,)++ # Override the container image for EDA+ config.sandbox.base_container_image = 'python:3.12-bookworm'+config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)agent_config.enable_prompt_extensions = Falsediff --git a/evaluation/benchmarks/agent_bench/run_infer.py b/evaluation/benchmarks/agent_bench/run_infer.pyindex 562df0024a..6a112c6abb 100644--- a/evaluation/benchmarks/agent_bench/run_infer.py+++ b/evaluation/benchmarks/agent_bench/run_infer.py@@ -17,8 +17,8 @@ from evaluation.utils.shared import (EvalMetadata,EvalOutput,compatibility_for_eval_history_pairs,- get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -41,19 +41,12 @@ from openhands.utils.async_utils import call_async_from_syncdef get_config(metadata: EvalMetadata,) -> OpenHandsConfig:- sandbox_config = get_default_sandbox_config_for_eval()- sandbox_config.base_container_image = 'python:3.12-slim'-- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- runtime=os.environ.get('RUNTIME', 'docker'),- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,- )+ # Create config with agent_bench-specific container image+ config = get_openhands_config_for_eval(metadata=metadata)++ # Override the container image for agent_bench+ config.sandbox.base_container_image = 'python:3.12-slim'+config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)agent_config.enable_prompt_extensions = Falsediff --git a/evaluation/benchmarks/aider_bench/run_infer.py b/evaluation/benchmarks/aider_bench/run_infer.pyindex 338315747d..f9eb65a9f8 100644--- a/evaluation/benchmarks/aider_bench/run_infer.py+++ b/evaluation/benchmarks/aider_bench/run_infer.py@@ -18,6 +18,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -50,15 +51,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.11-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,+ sandbox_config=sandbox_config,runtime=os.environ.get('RUNTIME', 'docker'),- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/biocoder/run_infer.py b/evaluation/benchmarks/biocoder/run_infer.pyindex 3dbc632a1b..39a01e2e4f 100644--- a/evaluation/benchmarks/biocoder/run_infer.py+++ b/evaluation/benchmarks/biocoder/run_infer.py@@ -16,6 +16,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -61,15 +62,10 @@ def get_config(sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = BIOCODER_BENCH_CONTAINER_IMAGE- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/bird/run_infer.py b/evaluation/benchmarks/bird/run_infer.pyindex 71886a0406..2f04011cc8 100644--- a/evaluation/benchmarks/bird/run_infer.py+++ b/evaluation/benchmarks/bird/run_infer.py@@ -19,6 +19,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -75,15 +76,10 @@ def get_config(sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/browsing_delegation/run_infer.py b/evaluation/benchmarks/browsing_delegation/run_infer.pyindex e6eb259d86..9f59817aed 100644--- a/evaluation/benchmarks/browsing_delegation/run_infer.py+++ b/evaluation/benchmarks/browsing_delegation/run_infer.py@@ -12,6 +12,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -40,14 +41,8 @@ def get_config()sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- workspace_base=None,- workspace_mount_path=None,+ config = get_openhands_config_for_eval(+ metadata=metadata, runtime='docker', sandbox_config=sandbox_config)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/commit0/run_infer.py b/evaluation/benchmarks/commit0/run_infer.pyindex bad4a735cd..fb125498c3 100644--- a/evaluation/benchmarks/commit0/run_infer.py+++ b/evaluation/benchmarks/commit0/run_infer.py@@ -17,6 +17,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -114,16 +115,11 @@ def get_config(sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = base_container_image- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,- enable_browser=RUN_WITH_BROWSING,+ config = get_openhands_config_for_eval(+ metadata=metadata,+ sandbox_config=sandbox_config,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ enable_browser=RUN_WITH_BROWSING,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/discoverybench/run_infer.py b/evaluation/benchmarks/discoverybench/run_infer.pyindex 448bc18e7b..1ffacc4543 100644--- a/evaluation/benchmarks/discoverybench/run_infer.py+++ b/evaluation/benchmarks/discoverybench/run_infer.py@@ -18,6 +18,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -65,15 +66,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/gaia/run_infer.py b/evaluation/benchmarks/gaia/run_infer.pyindex a7e69489c3..480df59bd2 100644--- a/evaluation/benchmarks/gaia/run_infer.py+++ b/evaluation/benchmarks/gaia/run_infer.py@@ -23,6 +23,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -60,15 +61,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'nikolaik/python-nodejs:python3.12-nodejs22'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,+ sandbox_config=sandbox_config,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,)config.set_llm_config(metadata.llm_config)if metadata.agent_config:diff --git a/evaluation/benchmarks/gorilla/run_infer.py b/evaluation/benchmarks/gorilla/run_infer.pyindex 87a71b4f00..932b0f6f75 100644--- a/evaluation/benchmarks/gorilla/run_infer.py+++ b/evaluation/benchmarks/gorilla/run_infer.py@@ -13,6 +13,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -43,15 +44,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/gpqa/run_infer.py b/evaluation/benchmarks/gpqa/run_infer.pyindex 28ac39588a..ce1cc3c8d0 100644--- a/evaluation/benchmarks/gpqa/run_infer.py+++ b/evaluation/benchmarks/gpqa/run_infer.py@@ -31,6 +31,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -64,15 +65,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/humanevalfix/run_infer.py b/evaluation/benchmarks/humanevalfix/run_infer.pyindex 8cf8b7efa1..7b882eab88 100644--- a/evaluation/benchmarks/humanevalfix/run_infer.py+++ b/evaluation/benchmarks/humanevalfix/run_infer.py@@ -24,6 +24,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -85,15 +86,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.pyindex 2aad6fb1b5..9363651545 100644--- a/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py+++ b/evaluation/benchmarks/lca_ci_build_repair/eval_infer.py@@ -16,6 +16,7 @@ import ruamel.yamlfrom evaluation.utils.shared import (EvalMetadata,get_default_sandbox_config_for_eval,+ get_openhands_config_for_eval,make_metadata,)from openhands.core.config import (@@ -37,15 +38,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/lca_ci_build_repair/run_infer.py b/evaluation/benchmarks/lca_ci_build_repair/run_infer.pyindex cbb53259ba..e6938301e0 100644--- a/evaluation/benchmarks/lca_ci_build_repair/run_infer.py+++ b/evaluation/benchmarks/lca_ci_build_repair/run_infer.py@@ -23,6 +23,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -48,15 +49,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/logic_reasoning/run_infer.py b/evaluation/benchmarks/logic_reasoning/run_infer.pyindex 23fa4e1d88..86fe82e6db 100644--- a/evaluation/benchmarks/logic_reasoning/run_infer.py+++ b/evaluation/benchmarks/logic_reasoning/run_infer.py@@ -11,6 +11,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -52,15 +53,10 @@ def get_config('$OH_INTERPRETER_PATH -m pip install scitools-pyke')- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/miniwob/run_infer.py b/evaluation/benchmarks/miniwob/run_infer.pyindex ef2fdc1412..a0e24a1bbb 100644--- a/evaluation/benchmarks/miniwob/run_infer.py+++ b/evaluation/benchmarks/miniwob/run_infer.py@@ -14,6 +14,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -58,15 +59,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'xingyaoww/od-eval-miniwob:v1.0'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime=os.environ.get('RUNTIME', 'docker'),- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/mint/run_infer.py b/evaluation/benchmarks/mint/run_infer.pyindex 72031e8d7b..890c4e5c0a 100644--- a/evaluation/benchmarks/mint/run_infer.py+++ b/evaluation/benchmarks/mint/run_infer.py@@ -16,6 +16,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -110,15 +111,10 @@ def get_config(f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}')- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/ml_bench/run_infer.py b/evaluation/benchmarks/ml_bench/run_infer.pyindex 32a8f76ca9..48e3d6785a 100644--- a/evaluation/benchmarks/ml_bench/run_infer.py+++ b/evaluation/benchmarks/ml_bench/run_infer.py@@ -27,6 +27,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -80,15 +81,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'public.ecr.aws/i5g0m1f6/ml-bench'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/multi_swe_bench/eval_infer.py b/evaluation/benchmarks/multi_swe_bench/eval_infer.pyindex 74364f2fe5..22fdcc764b 100644--- a/evaluation/benchmarks/multi_swe_bench/eval_infer.py+++ b/evaluation/benchmarks/multi_swe_bench/eval_infer.py@@ -23,6 +23,7 @@ from evaluation.utils.shared import (EvalMetadata,EvalOutput,get_default_sandbox_config_for_eval,+ get_openhands_config_for_eval,prepare_dataset,reset_logger_for_multiprocessing,run_evaluation,@@ -87,13 +88,9 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:dataset_name=metadata.dataset,instance_id=instance['instance_id'],)- config = OpenHandsConfig(- run_as_openhands=False,+ config = get_openhands_config_for_eval(runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)return configdiff --git a/evaluation/benchmarks/multi_swe_bench/run_infer.py b/evaluation/benchmarks/multi_swe_bench/run_infer.pyindex ca33f65298..db6524a8d5 100644--- a/evaluation/benchmarks/multi_swe_bench/run_infer.py+++ b/evaluation/benchmarks/multi_swe_bench/run_infer.py@@ -21,6 +21,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -341,16 +342,11 @@ def get_config(instance_id=instance['instance_id'],)- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,+ config = get_openhands_config_for_eval(+ metadata=metadata,enable_browser=RUN_WITH_BROWSING,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/nocode_bench/run_infer_nc.py b/evaluation/benchmarks/nocode_bench/run_infer_nc.pyindex 0022ed484d..3c3d40bdfc 100644--- a/evaluation/benchmarks/nocode_bench/run_infer_nc.py+++ b/evaluation/benchmarks/nocode_bench/run_infer_nc.py@@ -31,6 +31,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -174,15 +175,10 @@ def get_config(instance_id=instance['instance_id'],)- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(diff --git a/evaluation/benchmarks/scienceagentbench/run_infer.py b/evaluation/benchmarks/scienceagentbench/run_infer.pyindex 76b937308b..beabf3783f 100644--- a/evaluation/benchmarks/scienceagentbench/run_infer.py+++ b/evaluation/benchmarks/scienceagentbench/run_infer.py@@ -13,6 +13,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -64,16 +65,10 @@ def get_config(sandbox_config.base_container_image = ('docker.io/xingyaoww/openhands-eval-scienceagentbench')- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime=os.environ.get('RUNTIME', 'docker'),- max_budget_per_task=4,- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/swe_bench/eval_infer.py b/evaluation/benchmarks/swe_bench/eval_infer.pyindex 2fdc3e2e2f..46f3629be8 100644--- a/evaluation/benchmarks/swe_bench/eval_infer.py+++ b/evaluation/benchmarks/swe_bench/eval_infer.py@@ -19,6 +19,7 @@ from evaluation.utils.shared import (EvalMetadata,EvalOutput,get_default_sandbox_config_for_eval,+ get_openhands_config_for_eval,prepare_dataset,reset_logger_for_multiprocessing,run_evaluation,@@ -83,13 +84,9 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:dataset_name=metadata.dataset,instance_id=instance['instance_id'],)- config = OpenHandsConfig(- run_as_openhands=False,+ config = get_openhands_config_for_eval(runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)return configdiff --git a/evaluation/benchmarks/swe_bench/run_infer.py b/evaluation/benchmarks/swe_bench/run_infer.pyindex 4413f07d3b..3dbed38dbd 100644--- a/evaluation/benchmarks/swe_bench/run_infer.py+++ b/evaluation/benchmarks/swe_bench/run_infer.py@@ -32,6 +32,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -227,16 +228,11 @@ def get_config(instance_id=instance['instance_id'],)- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,+ config = get_openhands_config_for_eval(+ metadata=metadata,enable_browser=RUN_WITH_BROWSING,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(diff --git a/evaluation/benchmarks/swe_bench/run_localize.py b/evaluation/benchmarks/swe_bench/run_localize.pyindex 0c34991577..2f7f09912a 100644--- a/evaluation/benchmarks/swe_bench/run_localize.py+++ b/evaluation/benchmarks/swe_bench/run_localize.py@@ -20,6 +20,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -199,16 +200,11 @@ def get_config('REPO_PATH': f'/workspace/{workspace_dir_name}/',}- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,+ config = get_openhands_config_for_eval(+ metadata=metadata,enable_browser=RUN_WITH_BROWSING,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/testgeneval/eval_infer.py b/evaluation/benchmarks/testgeneval/eval_infer.pyindex 27aaaee1ac..1004a2d55f 100644--- a/evaluation/benchmarks/testgeneval/eval_infer.py+++ b/evaluation/benchmarks/testgeneval/eval_infer.py@@ -37,6 +37,7 @@ from evaluation.benchmarks.testgeneval.utils import load_testgeneval_datasetfrom evaluation.utils.shared import (EvalMetadata,EvalOutput,+ get_openhands_config_for_eval,prepare_dataset,reset_logger_for_multiprocessing,run_evaluation,@@ -58,20 +59,21 @@ def get_config(instance: pd.Series) -> OpenHandsConfig:f'Invalid container image for instance {instance["instance_id_swebench"]}.')logger.info(f'Using instance container image: {base_container_image}.')- return OpenHandsConfig(- run_as_openhands=False,- runtime=os.environ.get('RUNTIME', 'eventstream'),- sandbox=SandboxConfig(- base_container_image=base_container_image,- use_host_network=False,- timeout=1800,- api_key=os.environ.get('ALLHANDS_API_KEY'),- remote_runtime_api_url=os.environ.get(- 'SANDBOX_REMOTE_RUNTIME_API_URL', 'http://localhost:8000'- ),++ # Create custom sandbox config for testgeneval with specific requirements+ sandbox_config = SandboxConfig(+ base_container_image=base_container_image,+ use_host_network=False,+ timeout=1800, # Longer timeout than default (300)+ api_key=os.environ.get('ALLHANDS_API_KEY'),+ remote_runtime_api_url=os.environ.get(+ 'SANDBOX_REMOTE_RUNTIME_API_URL', 'http://localhost:8000'),- workspace_base=None,- workspace_mount_path=None,+ )++ return get_openhands_config_for_eval(+ sandbox_config=sandbox_config,+ runtime=os.environ.get('RUNTIME', 'docker'), # Different default runtime)diff --git a/evaluation/benchmarks/testgeneval/run_infer.py b/evaluation/benchmarks/testgeneval/run_infer.pyindex 39288ff537..c8171cca94 100644--- a/evaluation/benchmarks/testgeneval/run_infer.py+++ b/evaluation/benchmarks/testgeneval/run_infer.py@@ -25,6 +25,7 @@ from evaluation.utils.shared import (assert_and_raise,codeact_user_response,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -126,29 +127,26 @@ def get_config(f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.')- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,- runtime=os.environ.get('RUNTIME', 'eventstream'),- sandbox=SandboxConfig(- base_container_image=base_container_image,- enable_auto_lint=True,- use_host_network=False,- # large enough timeout, since some testcases take very long to run- timeout=300,- # Add platform to the sandbox config to solve issue 4401- platform='linux/amd64',- api_key=os.environ.get('ALLHANDS_API_KEY', None),- remote_runtime_api_url=os.environ.get(- 'SANDBOX_REMOTE_RUNTIME_API_URL', 'http://localhost:8000'- ),- keep_runtime_alive=False,- remote_runtime_init_timeout=3600,+ sandbox_config = SandboxConfig(+ base_container_image=base_container_image,+ enable_auto_lint=True,+ use_host_network=False,+ # large enough timeout, since some testcases take very long to run+ timeout=300,+ # Add platform to the sandbox config to solve issue 4401+ platform='linux/amd64',+ api_key=os.environ.get('ALLHANDS_API_KEY', None),+ remote_runtime_api_url=os.environ.get(+ 'SANDBOX_REMOTE_RUNTIME_API_URL', 'http://localhost:8000'),- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ keep_runtime_alive=False,+ remote_runtime_init_timeout=3600,+ )++ config = get_openhands_config_for_eval(+ metadata=metadata,+ sandbox_config=sandbox_config,+ runtime=os.environ.get('RUNTIME', 'docker'),)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/the_agent_company/run_infer.py b/evaluation/benchmarks/the_agent_company/run_infer.pyindex 513b02d612..ee768e7f6d 100644--- a/evaluation/benchmarks/the_agent_company/run_infer.py+++ b/evaluation/benchmarks/the_agent_company/run_infer.py@@ -12,7 +12,10 @@ import tempfileimport yamlfrom browsing import pre_login-from evaluation.utils.shared import get_default_sandbox_config_for_eval+from evaluation.utils.shared import (+ get_default_sandbox_config_for_eval,+ get_openhands_config_for_eval,+)from openhands.controller.state.state import Statefrom openhands.core.config import (LLMConfig,@@ -42,19 +45,17 @@ def get_config(sandbox_config.enable_auto_lint = True# If the web services are running on the host machine, this must be set to Truesandbox_config.use_host_network = True- config = OpenHandsConfig(- run_as_openhands=False,- max_budget_per_task=4,+ config = get_openhands_config_for_eval(max_iterations=100,- save_trajectory_path=os.path.join(- mount_path_on_host, f'traj_{task_short_name}.json'- ),- sandbox=sandbox_config,# we mount trajectories path so that trajectories, generated by OpenHands# controller, can be accessible to the evaluator file in the runtime container+ sandbox_config=sandbox_config,workspace_mount_path=mount_path_on_host,- workspace_mount_path_in_sandbox='/outputs',)+ config.save_trajectory_path = os.path.join(+ mount_path_on_host, f'traj_{task_short_name}.json'+ )+ config.max_budget_per_task = 4config.set_llm_config(llm_config)if agent_config:config.set_agent_config(agent_config)diff --git a/evaluation/benchmarks/toolqa/run_infer.py b/evaluation/benchmarks/toolqa/run_infer.pyindex 8353f76f77..3fa078648d 100644--- a/evaluation/benchmarks/toolqa/run_infer.py+++ b/evaluation/benchmarks/toolqa/run_infer.py@@ -12,6 +12,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -44,15 +45,10 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.base_container_image = 'python:3.12-bookworm'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/benchmarks/visual_swe_bench/run_infer.py b/evaluation/benchmarks/visual_swe_bench/run_infer.pyindex 215f7933b1..ca096d9e19 100644--- a/evaluation/benchmarks/visual_swe_bench/run_infer.py+++ b/evaluation/benchmarks/visual_swe_bench/run_infer.py@@ -20,6 +20,7 @@ from evaluation.utils.shared import (codeact_user_response,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,is_fatal_evaluation_error,make_metadata,prepare_dataset,@@ -160,16 +161,11 @@ def get_config(instance_id=instance['instance_id'],)- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,- max_iterations=metadata.max_iterations,+ config = get_openhands_config_for_eval(+ metadata=metadata,enable_browser=RUN_WITH_BROWSING,runtime=os.environ.get('RUNTIME', 'docker'),- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/visualwebarena/run_infer.py b/evaluation/benchmarks/visualwebarena/run_infer.pyindex ef45663e6a..7c0695d585 100644--- a/evaluation/benchmarks/visualwebarena/run_infer.py+++ b/evaluation/benchmarks/visualwebarena/run_infer.py@@ -13,6 +13,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -73,16 +74,10 @@ def get_config('VWA_WIKIPEDIA': f'{base_url}:8888','VWA_HOMEPAGE': f'{base_url}:4399',}- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,- attach_to_existing=True,+ sandbox_config=sandbox_config,)config.set_llm_config(update_llm_config_for_completions_logging(diff --git a/evaluation/benchmarks/webarena/run_infer.py b/evaluation/benchmarks/webarena/run_infer.pyindex 316dba63de..273a137124 100644--- a/evaluation/benchmarks/webarena/run_infer.py+++ b/evaluation/benchmarks/webarena/run_infer.py@@ -13,6 +13,7 @@ from evaluation.utils.shared import (compatibility_for_eval_history_pairs,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -65,15 +66,10 @@ def get_config('MAP': f'{base_url}:3000','HOMEPAGE': f'{base_url}:4399',}- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime='docker',- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,+ sandbox_config=sandbox_config,)config.set_llm_config(metadata.llm_config)agent_config = config.get_agent_config(metadata.agent_class)diff --git a/evaluation/integration_tests/run_infer.py b/evaluation/integration_tests/run_infer.pyindex c493cc173a..c2ccf54bc9 100644--- a/evaluation/integration_tests/run_infer.py+++ b/evaluation/integration_tests/run_infer.py@@ -10,6 +10,7 @@ from evaluation.utils.shared import (EvalOutput,get_default_sandbox_config_for_eval,get_metrics,+ get_openhands_config_for_eval,make_metadata,prepare_dataset,reset_logger_for_multiprocessing,@@ -45,18 +46,12 @@ def get_config() -> OpenHandsConfig:sandbox_config = get_default_sandbox_config_for_eval()sandbox_config.platform = 'linux/amd64'- config = OpenHandsConfig(- default_agent=metadata.agent_class,- run_as_openhands=False,+ config = get_openhands_config_for_eval(+ metadata=metadata,runtime=os.environ.get('RUNTIME', 'docker'),- max_iterations=metadata.max_iterations,- sandbox=sandbox_config,- # do not mount workspace- workspace_base=None,- workspace_mount_path=None,- # debug- debug=True,+ sandbox_config=sandbox_config,)+ config.debug = Trueconfig.set_llm_config(update_llm_config_for_completions_logging(metadata.llm_config, metadata.eval_output_dir, instance_iddiff --git a/evaluation/utils/shared.py b/evaluation/utils/shared.pyindex 76ed563ec2..a3d9c125af 100644--- a/evaluation/utils/shared.py+++ b/evaluation/utils/shared.py@@ -703,3 +703,79 @@ def get_default_sandbox_config_for_eval() -> SandboxConfig:remote_runtime_enable_retries=True,remote_runtime_class='sysbox',)+++def get_openhands_config_for_eval(+ metadata: EvalMetadata | None = None,+ sandbox_config: SandboxConfig | None = None,+ runtime: str | None = None,+ max_iterations: int | None = None,+ default_agent: str | None = None,+ enable_browser: bool = False,+ workspace_base: str | None = None,+ workspace_mount_path: str | None = None,+):+ """Create an OpenHandsConfig with common patterns used across evaluation scripts.++ This function provides a standardized way to create OpenHands configurations+ for evaluation runs, with sensible defaults that match the patterns used in+ most run_infer.py scripts. Individual evaluation scripts can override specific+ attributes as needed.++ Args:+ metadata: EvalMetadata containing agent class, max iterations, etc.+ sandbox_config: Custom sandbox config. If None, uses get_default_sandbox_config_for_eval()+ runtime: Runtime type. If None, uses environment RUNTIME or 'docker'+ max_iterations: Max iterations for the agent. If None, uses metadata.max_iterations+ default_agent: Agent class name. If None, uses metadata.agent_class+ enable_browser: Whether to enable browser functionality+ workspace_base: Workspace base path. Defaults to None+ workspace_mount_path: Workspace mount path. Defaults to None++ Returns:+ OpenHandsConfig: Configured for evaluation with eval-specific overrides applied+ """+ # Defer import to avoid circular imports at module load time+ from openhands.core.config.openhands_config import (+ OpenHandsConfig as _OHConfig, # type: ignore+ )++ # Use provided sandbox config or get default+ if sandbox_config is None:+ sandbox_config = get_default_sandbox_config_for_eval()++ # Extract values from metadata if provided+ if metadata is not None:+ if max_iterations is None:+ max_iterations = metadata.max_iterations+ if default_agent is None:+ default_agent = metadata.agent_class++ # Use environment runtime or default+ if runtime is None:+ runtime = os.environ.get('RUNTIME', 'docker')++ # Provide sensible defaults if still None+ if default_agent is None:+ default_agent = 'CodeActAgent'+ if max_iterations is None:+ max_iterations = 50++ # Always use repo-local .eval_sessions directory (absolute path)+ eval_store = os.path.abspath(os.path.join(os.getcwd(), '.eval_sessions'))++ # Create the base config with evaluation-specific overrides+ config = _OHConfig(+ default_agent=default_agent,+ run_as_openhands=False,+ runtime=runtime,+ max_iterations=max_iterations,+ enable_browser=enable_browser,+ sandbox=sandbox_config,+ workspace_base=workspace_base,+ workspace_mount_path=workspace_mount_path,+ file_store='local',+ file_store_path=eval_store,+ )++ return configdiff --git a/tests/unit/evaluation/test_eval_file_store_path.py b/tests/unit/evaluation/test_eval_file_store_path.pynew file mode 100644index 0000000000..1314612729--- /dev/null+++ b/tests/unit/evaluation/test_eval_file_store_path.py@@ -0,0 +1,26 @@+import os+from pathlib import Path++from evaluation.utils.shared import get_openhands_config_for_eval+++def test_eval_file_store_defaults_to_repo_local(tmp_path, monkeypatch):+ prev_cwd = Path.cwd()+ try:+ os.chdir(tmp_path)+ cfg = get_openhands_config_for_eval()+ assert Path(cfg.file_store_path) == (tmp_path / '.eval_sessions').resolve()+ assert cfg.file_store == 'local'+ finally:+ os.chdir(prev_cwd)+++def test_eval_file_store_is_hard_coded_repo_local(tmp_path):+ prev_cwd = Path.cwd()+ try:+ os.chdir(tmp_path)+ cfg = get_openhands_config_for_eval()+ assert Path(cfg.file_store_path) == (tmp_path / '.eval_sessions').resolve()+ assert cfg.file_store == 'local'+ finally:+ os.chdir(prev_cwd)