A programming framework for agentic AI
fixing default metric for regression + change verbosity for transformers (#397)
Xueqing Liuc1b5cb534890
diff --git a/flaml/automl.py b/flaml/automl.pyindex 47b9169e9..6f0ef0d68 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -470,7 +470,6 @@ class AutoML(BaseEstimator):'classification', 'regression', 'ts_forecast', 'rank','seq-classification', 'seq-regression', 'summarization'.n_jobs: An integer of the number of threads for training.- gpu_per_trial: A float of the number of gpus per trial, only used by TransformersEstimator.log_file_name: A string of the log file name. To disable logging,set it to be an empty string "".estimator_list: A list of strings for estimator names, or 'auto'@@ -576,7 +575,6 @@ class AutoML(BaseEstimator):settings["time_budget"] = settings.get("time_budget", 60)settings["task"] = settings.get("task", "classification")settings["n_jobs"] = settings.get("n_jobs", -1)- settings["gpu_per_trial"] = settings.get("gpu_per_trial", 0)settings["eval_method"] = settings.get("eval_method", "auto")settings["split_ratio"] = settings.get("split_ratio", SPLIT_RATIO)settings["n_splits"] = settings.get("n_splits", N_SPLITS)@@ -1306,7 +1304,7 @@ class AutoML(BaseEstimator):split_type=None,groups=None,n_jobs=-1,- gpu_per_trial=0,+ # gpu_per_trial=0,train_best=True,train_full=False,record_id=-1,@@ -1358,7 +1356,6 @@ class AutoML(BaseEstimator):for training data.n_jobs: An integer of the number of threads for training. Use allavailable resources when n_jobs == -1.- gpu_per_trial: A float of the number of gpus per trial. Only used by TransformersEstimator.train_best: A boolean of whether to train the best config in thetime budget; if false, train the last config in the budget.train_full: A boolean of whether to train on the full data. If true,@@ -1451,9 +1448,9 @@ class AutoML(BaseEstimator):import osself._state.resources_per_trial = (- {"cpu": os.cpu_count(), "gpu": gpu_per_trial}+ {"cpu": os.cpu_count(), "gpu": fit_kwargs.get("gpu_per_trial", 0)}if self._state.n_jobs < 0- else {"cpu": self._state.n_jobs, "gpu": gpu_per_trial}+ else {"cpu": self._state.n_jobs, "gpu": fit_kwargs.get("gpu_per_trial", 0)})self._trained_estimator = self._state._train_with_config(best_estimator,@@ -1711,7 +1708,7 @@ class AutoML(BaseEstimator):metric=None,task=None,n_jobs=None,- gpu_per_trial=0,+ # gpu_per_trial=0,log_file_name=None,estimator_list=None,time_budget=None,@@ -1807,7 +1804,6 @@ class AutoML(BaseEstimator):'classification', 'regression', 'ts_forecast', 'rank','seq-classification', 'seq-regression', 'summarization'n_jobs: An integer of the number of threads for training.- gpu_per_trial: A float of the number of gpus per trial, only used by TransformersEstimator.log_file_name: A string of the log file name. To disable logging,set it to be an empty string "".estimator_list: A list of strings for estimator names, or 'auto'@@ -1918,8 +1914,10 @@ class AutoML(BaseEstimator):datasets, but will incur more overhead in time. Only use it ifyou run into OOM failures.**fit_kwargs: Other key word arguments to pass to fit() function of- the searched learners, such as sample_weight. Include period as- a key word argument for 'ts_forecast' task.+ the searched learners, such as sample_weight. Include:+ period: int | forecast horizon for 'ts_forecast' task.+ gpu_per_trial: float, default = 0 | A float of the number of gpus per trial,+ only used by TransformersEstimator."""self._state._start_time_flag = self._start_time_flag = time.time()@@ -1927,11 +1925,7 @@ class AutoML(BaseEstimator):self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor"time_budget = time_budget or self._settings.get("time_budget")n_jobs = n_jobs or self._settings.get("n_jobs")- gpu_per_trial = (- self._settings.get("gpu_per_trial")- if gpu_per_trial is None- else gpu_per_trial- )+ gpu_per_trial = fit_kwargs.get("gpu_per_trial", 0)eval_method = eval_method or self._settings.get("eval_method")split_ratio = split_ratio or self._settings.get("split_ratio")n_splits = n_splits or self._settings.get("n_splits")@@ -2081,6 +2075,7 @@ class AutoML(BaseEstimator):if _is_nlp_task(self._state.task):self._state.fit_kwargs["metric"] = metricself._state.fit_kwargs["use_ray"] = self._use_ray+ self._state.fit_kwargs["gpu_per_trial"] = gpu_per_trialself._state.metric = metricdiff --git a/flaml/model.py b/flaml/model.pyindex 045ba11f8..d5f57bda9 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -580,6 +580,8 @@ class TransformersEstimator(BaseEstimator):setattr(self._trainer, "_use_ray", self.use_ray)if self._task in NLG_TASKS:setattr(self._trainer, "_is_seq2seq", True)+ if kwargs.get("gpu_per_trial"):+ self._trainer.args._n_gpu = kwargs.get("gpu_per_trial")self._trainer.train()self.params[self.ITER_HP] = self._trainer.state.global_stepdiff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.pyindex c714c054a..144a1e560 100644--- a/flaml/nlp/utils.py+++ b/flaml/nlp/utils.py@@ -18,7 +18,7 @@ def load_default_huggingface_metric_for_task(task):if task == SEQCLASSIFICATION:return "accuracy"elif task == SEQREGRESSION:- return "rmse"+ return "r2"elif task == SUMMARIZATION:return "rouge"elif task == MULTICHOICECLASSIFICATION:@@ -400,6 +400,10 @@ def get_trial_fold_name(local_dir, trial_config, trial_id):def load_model(checkpoint_path, task, num_labels, per_model_config=None):+ import transformers++ transformers.logging.set_verbosity_error()+from transformers import AutoConfigfrom .huggingface.switch_head_auto import (AutoSeqClassificationHead,