Skip to content

A programming framework for agentic AI

fixing default metric for regression + change verbosity for transformers (#397)

Xueqing Liuc1b5cb534890
diff --git a/flaml/automl.py b/flaml/automl.py
index 47b9169e9..6f0ef0d68 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -470,7 +470,6 @@ class AutoML(BaseEstimator):
'classification', 'regression', 'ts_forecast', 'rank',
'seq-classification', 'seq-regression', 'summarization'.
n_jobs: An integer of the number of threads for training.
- gpu_per_trial: A float of the number of gpus per trial, only used by TransformersEstimator.
log_file_name: A string of the log file name. To disable logging,
set it to be an empty string "".
estimator_list: A list of strings for estimator names, or 'auto'
@@ -576,7 +575,6 @@ class AutoML(BaseEstimator):
settings["time_budget"] = settings.get("time_budget", 60)
settings["task"] = settings.get("task", "classification")
settings["n_jobs"] = settings.get("n_jobs", -1)
- settings["gpu_per_trial"] = settings.get("gpu_per_trial", 0)
settings["eval_method"] = settings.get("eval_method", "auto")
settings["split_ratio"] = settings.get("split_ratio", SPLIT_RATIO)
settings["n_splits"] = settings.get("n_splits", N_SPLITS)
@@ -1306,7 +1304,7 @@ class AutoML(BaseEstimator):
split_type=None,
groups=None,
n_jobs=-1,
- gpu_per_trial=0,
+ # gpu_per_trial=0,
train_best=True,
train_full=False,
record_id=-1,
@@ -1358,7 +1356,6 @@ class AutoML(BaseEstimator):
for training data.
n_jobs: An integer of the number of threads for training. Use all
available resources when n_jobs == -1.
- gpu_per_trial: A float of the number of gpus per trial. Only used by TransformersEstimator.
train_best: A boolean of whether to train the best config in the
time budget; if false, train the last config in the budget.
train_full: A boolean of whether to train on the full data. If true,
@@ -1451,9 +1448,9 @@ class AutoML(BaseEstimator):
import os
self._state.resources_per_trial = (
- {"cpu": os.cpu_count(), "gpu": gpu_per_trial}
+ {"cpu": os.cpu_count(), "gpu": fit_kwargs.get("gpu_per_trial", 0)}
if self._state.n_jobs < 0
- else {"cpu": self._state.n_jobs, "gpu": gpu_per_trial}
+ else {"cpu": self._state.n_jobs, "gpu": fit_kwargs.get("gpu_per_trial", 0)}
)
self._trained_estimator = self._state._train_with_config(
best_estimator,
@@ -1711,7 +1708,7 @@ class AutoML(BaseEstimator):
metric=None,
task=None,
n_jobs=None,
- gpu_per_trial=0,
+ # gpu_per_trial=0,
log_file_name=None,
estimator_list=None,
time_budget=None,
@@ -1807,7 +1804,6 @@ class AutoML(BaseEstimator):
'classification', 'regression', 'ts_forecast', 'rank',
'seq-classification', 'seq-regression', 'summarization'
n_jobs: An integer of the number of threads for training.
- gpu_per_trial: A float of the number of gpus per trial, only used by TransformersEstimator.
log_file_name: A string of the log file name. To disable logging,
set it to be an empty string "".
estimator_list: A list of strings for estimator names, or 'auto'
@@ -1918,8 +1914,10 @@ class AutoML(BaseEstimator):
datasets, but will incur more overhead in time. Only use it if
you run into OOM failures.
**fit_kwargs: Other key word arguments to pass to fit() function of
- the searched learners, such as sample_weight. Include period as
- a key word argument for 'ts_forecast' task.
+ the searched learners, such as sample_weight. Include:
+ period: int | forecast horizon for 'ts_forecast' task.
+ gpu_per_trial: float, default = 0 | A float of the number of gpus per trial,
+ only used by TransformersEstimator.
"""
self._state._start_time_flag = self._start_time_flag = time.time()
@@ -1927,11 +1925,7 @@ class AutoML(BaseEstimator):
self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor"
time_budget = time_budget or self._settings.get("time_budget")
n_jobs = n_jobs or self._settings.get("n_jobs")
- gpu_per_trial = (
- self._settings.get("gpu_per_trial")
- if gpu_per_trial is None
- else gpu_per_trial
- )
+ gpu_per_trial = fit_kwargs.get("gpu_per_trial", 0)
eval_method = eval_method or self._settings.get("eval_method")
split_ratio = split_ratio or self._settings.get("split_ratio")
n_splits = n_splits or self._settings.get("n_splits")
@@ -2081,6 +2075,7 @@ class AutoML(BaseEstimator):
if _is_nlp_task(self._state.task):
self._state.fit_kwargs["metric"] = metric
self._state.fit_kwargs["use_ray"] = self._use_ray
+ self._state.fit_kwargs["gpu_per_trial"] = gpu_per_trial
self._state.metric = metric
diff --git a/flaml/model.py b/flaml/model.py
index 045ba11f8..d5f57bda9 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -580,6 +580,8 @@ class TransformersEstimator(BaseEstimator):
setattr(self._trainer, "_use_ray", self.use_ray)
if self._task in NLG_TASKS:
setattr(self._trainer, "_is_seq2seq", True)
+ if kwargs.get("gpu_per_trial"):
+ self._trainer.args._n_gpu = kwargs.get("gpu_per_trial")
self._trainer.train()
self.params[self.ITER_HP] = self._trainer.state.global_step
diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.py
index c714c054a..144a1e560 100644
--- a/flaml/nlp/utils.py
+++ b/flaml/nlp/utils.py
@@ -18,7 +18,7 @@ def load_default_huggingface_metric_for_task(task):
if task == SEQCLASSIFICATION:
return "accuracy"
elif task == SEQREGRESSION:
- return "rmse"
+ return "r2"
elif task == SUMMARIZATION:
return "rouge"
elif task == MULTICHOICECLASSIFICATION:
@@ -400,6 +400,10 @@ def get_trial_fold_name(local_dir, trial_config, trial_id):
def load_model(checkpoint_path, task, num_labels, per_model_config=None):
+ import transformers
+
+ transformers.logging.set_verbosity_error()
+
from transformers import AutoConfig
from .huggingface.switch_head_auto import (
AutoSeqClassificationHead,