🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
automatically update args.low_is_better for mle-bench
Yizhou Chi852fbc58ee9e
diff --git a/expo/data/custom_task.py b/expo/data/custom_task.pyindex c4dd0012c..f3cd433f5 100644--- a/expo/data/custom_task.py+++ b/expo/data/custom_task.py@@ -35,6 +35,15 @@ def get_mle_task_id(dataset_dir):return dataset_dir.split("/")[-3]+def get_mle_is_lower_better(task):+ from mlebench.data import get_leaderboard+ from mlebench.registry import registry++ competition = registry.get_competition(task)+ competition_leaderboard = get_leaderboard(competition)+ return competition.grader.is_lower_better(competition_leaderboard)++def get_mle_bench_requirements(dataset_dir, data_config, special_instruction, obfuscated=False):work_dir = data_config["work_dir"]task = get_mle_task_id(dataset_dir)diff --git a/expo/run_experiment.py b/expo/run_experiment.pyindex be891814d..7b49e6738 100644--- a/expo/run_experiment.py+++ b/expo/run_experiment.py@@ -1,7 +1,7 @@import argparseimport asyncio-from expo.data.custom_task import get_mle_task_id+from expo.data.custom_task import get_mle_is_lower_better, get_mle_task_idfrom expo.experimenter.aug import AugExperimenterfrom expo.experimenter.autogluon import GluonExperimenterfrom expo.experimenter.autosklearn import AutoSklearnExperimenter@@ -33,6 +33,8 @@ def get_args(cmd=True):args.eval_func = "mlebench"args.from_scratch = Trueargs.task = get_mle_task_id(args.custom_dataset_dir)+ args.low_is_better = get_mle_is_lower_better(args.task)+ print("low_is_better:", args.low_is_better)return args