Skip to content

A programming framework for agentic AI

Refactor into automl subpackage (#809)

Mark Harley44ddf9e10465
diff --git a/README.md b/README.md
index 9290aa329..d2590a6cc 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,7 @@ tune.run(evaluation_function, config={…}, low_cost_partial_config={…}, time_
```python
from flaml.default import LGBMRegressor
+
# Use LGBMRegressor in the same way as you use lightgbm.LGBMRegressor.
estimator = LGBMRegressor()
# The hyperparameters are automatically set according to the training data.
diff --git a/flaml/__init__.py b/flaml/__init__.py
index ecd80cf69..161a81cea 100644
--- a/flaml/__init__.py
+++ b/flaml/__init__.py
@@ -1,5 +1,5 @@
-from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch
from flaml.automl import AutoML, logger_formatter
+from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch
from flaml.onlineml.autovw import AutoVW
from flaml.version import __version__
import logging
diff --git a/flaml/automl/__init__.py b/flaml/automl/__init__.py
new file mode 100644
index 000000000..b93cd5b14
--- /dev/null
+++ b/flaml/automl/__init__.py
@@ -0,0 +1,3 @@
+from flaml.automl.automl import AutoML, AutoMLState, SearchState, logger_formatter, size
+
+__all__ = ["AutoML", "AutoMLState", "SearchState", "logger_formatter", "size"]
diff --git a/flaml/automl.py b/flaml/automl/automl.py
similarity index 99%
rename from flaml/automl.py
rename to flaml/automl/automl.py
index 664e712cf..794bb1748 100644
--- a/flaml/automl.py
+++ b/flaml/automl/automl.py
@@ -22,13 +22,13 @@ from sklearn.base import BaseEstimator
import pandas as pd
import logging
import json
-from .ml import (
+from flaml.automl.ml import (
compute_estimator,
train_estimator,
get_estimator_class,
get_classification_objective,
)
-from .config import (
+from flaml.config import (
MIN_SAMPLE_TRAIN,
MEM_THRES,
RANDOM_SEED,
@@ -38,7 +38,7 @@ from .config import (
N_SPLITS,
SAMPLE_MULTIPLY_FACTOR,
)
-from .data import (
+from flaml.automl.data import (
concat,
CLASSIFICATION,
TOKENCLASSIFICATION,
@@ -50,9 +50,10 @@ from .data import (
_is_nlp_task,
NLG_TASKS,
)
-from . import tune
-from .training_log import training_log_reader, training_log_writer
-from flaml.default.suggest import suggest_learner
+from flaml import tune
+from flaml.automl.training_log import training_log_reader, training_log_writer
+from flaml.default import suggest_learner
+from flaml.version import __version__ as flaml_version
logger = logging.getLogger(__name__)
logger_formatter = logging.Formatter(
@@ -78,7 +79,7 @@ class SearchState:
)
def valid_starting_point_one_dim(self, value_one_dim, domain_one_dim):
- from .tune.space import sample
+ from flaml.tune.space import sample
"""
For each hp in the starting point, check the following 3 conditions:
@@ -500,7 +501,7 @@ class AutoML(BaseEstimator):
"""
- from .version import __version__
+ __version__ = flaml_version
def __init__(self, **settings):
"""Constructor.
@@ -640,7 +641,7 @@ class AutoML(BaseEstimator):
seed: int or None, default=None | The random seed for hpo.
n_concurrent_trials: [Experimental] int, default=1 | The number of
concurrent trials. When n_concurrent_trials > 1, flaml performes
- [parallel tuning](../Use-Cases/Task-Oriented-AutoML#parallel-tuning)
+ [parallel tuning](../../Use-Cases/Task-Oriented-AutoML#parallel-tuning)
and installation of ray is required: `pip install flaml[ray]`.
keep_search_state: boolean, default=False | Whether to keep data needed
for model search after fit(). By default the state is deleted for
@@ -671,7 +672,7 @@ class AutoML(BaseEstimator):
the metrics_to_log dictionary returned by a customized metric function.
The customized metric function shall be provided via the `metric` key word
argument of the fit() function or the automl constructor.
- Find an example in the 4th constraint type in this [doc](../Use-Cases/Task-Oriented-AutoML#constraint).
+ Find an example in the 4th constraint type in this [doc](../../Use-Cases/Task-Oriented-AutoML#constraint).
If `pred_time_limit` is provided as one of keyword arguments to fit() function or
the automl constructor, flaml will automatically (and under the hood)
add it as an additional element in the metric_constraints. Essentially 'pred_time_limit'
@@ -2336,7 +2337,7 @@ class AutoML(BaseEstimator):
seed: int or None, default=None | The random seed for hpo.
n_concurrent_trials: [Experimental] int, default=1 | The number of
concurrent trials. When n_concurrent_trials > 1, flaml performes
- [parallel tuning](../Use-Cases/Task-Oriented-AutoML#parallel-tuning)
+ [parallel tuning](../../Use-Cases/Task-Oriented-AutoML#parallel-tuning)
and installation of ray is required: `pip install flaml[ray]`.
keep_search_state: boolean, default=False | Whether to keep data needed
for model search after fit(). By default the state is deleted for
@@ -2699,7 +2700,9 @@ class AutoML(BaseEstimator):
)
if "auto" == metric:
if _is_nlp_task(self._state.task):
- from .nlp.utils import load_default_huggingface_metric_for_task
+ from flaml.automl.nlp.utils import (
+ load_default_huggingface_metric_for_task,
+ )
metric = load_default_huggingface_metric_for_task(self._state.task)
elif "binary" in self._state.task:
@@ -2734,7 +2737,7 @@ class AutoML(BaseEstimator):
]:
return True, f"1-{metric}"
if _is_nlp_task(task):
- from .ml import huggingface_metric_to_mode
+ from flaml.automl.ml import huggingface_metric_to_mode
if (
metric in huggingface_metric_to_mode
diff --git a/flaml/automl/data.py b/flaml/automl/data.py
new file mode 100644
index 000000000..01f27d14e
--- /dev/null
+++ b/flaml/automl/data.py
@@ -0,0 +1,485 @@
+# !
+# * Copyright (c) Microsoft Corporation. All rights reserved.
+# * Licensed under the MIT License. See LICENSE file in the
+# * project root for license information.
+import numpy as np
+from scipy.sparse import vstack, issparse
+import pandas as pd
+from pandas import DataFrame, Series
+
+from flaml.automl.training_log import training_log_reader
+
+from datetime import datetime
+from typing import Union
+
+# TODO: if your task is not specified in here, define your task as an all-capitalized word
+SEQCLASSIFICATION = "seq-classification"
+MULTICHOICECLASSIFICATION = "multichoice-classification"
+TOKENCLASSIFICATION = "token-classification"
+CLASSIFICATION = (
+ "binary",
+ "multiclass",
+ "classification",
+ SEQCLASSIFICATION,
+ MULTICHOICECLASSIFICATION,
+ TOKENCLASSIFICATION,
+)
+SEQREGRESSION = "seq-regression"
+REGRESSION = ("regression", SEQREGRESSION)
+TS_FORECASTREGRESSION = (
+ "forecast",
+ "ts_forecast",
+ "ts_forecast_regression",
+)
+TS_FORECASTCLASSIFICATION = "ts_forecast_classification"
+TS_FORECASTPANEL = "ts_forecast_panel"
+TS_FORECAST = (
+ *TS_FORECASTREGRESSION,
+ TS_FORECASTCLASSIFICATION,
+ TS_FORECASTPANEL,
+)
+TS_TIMESTAMP_COL = "ds"
+TS_VALUE_COL = "y"
+SUMMARIZATION = "summarization"
+NLG_TASKS = (SUMMARIZATION,)
+NLU_TASKS = (
+ SEQREGRESSION,
+ SEQCLASSIFICATION,
+ MULTICHOICECLASSIFICATION,
+ TOKENCLASSIFICATION,
+)
+
+
+def _is_nlp_task(task):
+ if task in NLU_TASKS or task in NLG_TASKS:
+ return True
+ else:
+ return False
+
+
+def load_openml_dataset(
+ dataset_id, data_dir=None, random_state=0, dataset_format="dataframe"
+):
+ """Load dataset from open ML.
+
+ If the file is not cached locally, download it from open ML.
+
+ Args:
+ dataset_id: An integer of the dataset id in openml.
+ data_dir: A string of the path to store and load the data.
+ random_state: An integer of the random seed for splitting data.
+ dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'.
+ Can choose from ['dataframe', 'array'].
+ If 'dataframe', the returned dataset will be a Pandas DataFrame.
+ If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix.
+
+ Returns:
+ X_train: Training data.
+ X_test: Test data.
+ y_train: A series or array of labels for training data.
+ y_test: A series or array of labels for test data.
+ """
+ import os
+ import openml
+ import pickle
+ from sklearn.model_selection import train_test_split
+
+ filename = "openml_ds" + str(dataset_id) + ".pkl"
+ filepath = os.path.join(data_dir, filename)
+ if os.path.isfile(filepath):
+ print("load dataset from", filepath)
+ with open(filepath, "rb") as f:
+ dataset = pickle.load(f)
+ else:
+ print("download dataset from openml")
+ dataset = openml.datasets.get_dataset(dataset_id)
+ if not os.path.exists(data_dir):
+ os.makedirs(data_dir)
+ with open(filepath, "wb") as f:
+ pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
+ print("Dataset name:", dataset.name)
+ try:
+ X, y, *__ = dataset.get_data(
+ target=dataset.default_target_attribute, dataset_format=dataset_format
+ )
+ except ValueError:
+ from sklearn.datasets import fetch_openml
+
+ X, y = fetch_openml(data_id=dataset_id, return_X_y=True)
+ X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state)
+ print(
+ "X_train.shape: {}, y_train.shape: {};\nX_test.shape: {}, y_test.shape: {}".format(
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+ )
+ return X_train, X_test, y_train, y_test
+
+
+def load_openml_task(task_id, data_dir):
+ """Load task from open ML.
+
+ Use the first fold of the task.
+ If the file is not cached locally, download it from open ML.
+
+ Args:
+ task_id: An integer of the task id in openml.
+ data_dir: A string of the path to store and load the data.
+
+ Returns:
+ X_train: A dataframe of training data.
+ X_test: A dataframe of test data.
+ y_train: A series of labels for training data.
+ y_test: A series of labels for test data.
+ """
+ import os
+ import openml
+ import pickle
+
+ task = openml.tasks.get_task(task_id)
+ filename = "openml_task" + str(task_id) + ".pkl"
+ filepath = os.path.join(data_dir, filename)
+ if os.path.isfile(filepath):
+ print("load dataset from", filepath)
+ with open(filepath, "rb") as f:
+ dataset = pickle.load(f)
+ else:
+ print("download dataset from openml")
+ dataset = task.get_dataset()
+ with open(filepath, "wb") as f:
+ pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
+ X, y, _, _ = dataset.get_data(task.target_name)
+ train_indices, test_indices = task.get_train_test_split_indices(
+ repeat=0,
+ fold=0,
+ sample=0,
+ )
+ X_train = X.iloc[train_indices]
+ y_train = y[train_indices]
+ X_test = X.iloc[test_indices]
+ y_test = y[test_indices]
+ print(
+ "X_train.shape: {}, y_train.shape: {},\nX_test.shape: {}, y_test.shape: {}".format(
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+ )
+ return X_train, X_test, y_train, y_test
+
+
+def get_output_from_log(filename, time_budget):
+ """Get output from log file.
+
+ Args:
+ filename: A string of the log file name.
+ time_budget: A float of the time budget in seconds.
+
+ Returns:
+ search_time_list: A list of the finished time of each logged iter.
+ best_error_list: A list of the best validation error after each logged iter.
+ error_list: A list of the validation error of each logged iter.
+ config_list: A list of the estimator, sample size and config of each logged iter.
+ logged_metric_list: A list of the logged metric of each logged iter.
+ """
+
+ best_config = None
+ best_learner = None
+ best_val_loss = float("+inf")
+
+ search_time_list = []
+ config_list = []
+ best_error_list = []
+ error_list = []
+ logged_metric_list = []
+ best_config_list = []
+ with training_log_reader(filename) as reader:
+ for record in reader.records():
+ time_used = record.wall_clock_time
+ val_loss = record.validation_loss
+ config = record.config
+ learner = record.learner.split("_")[0]
+ sample_size = record.sample_size
+ metric = record.logged_metric
+
+ if time_used < time_budget and np.isfinite(val_loss):
+ if val_loss < best_val_loss:
+ best_val_loss = val_loss
+ best_config = config
+ best_learner = learner
+ best_config_list.append(best_config)
+ search_time_list.append(time_used)
+ best_error_list.append(best_val_loss)
+ logged_metric_list.append(metric)
+ error_list.append(val_loss)
+ config_list.append(
+ {
+ "Current Learner": learner,
+ "Current Sample": sample_size,
+ "Current Hyper-parameters": record.config,
+ "Best Learner": best_learner,
+ "Best Hyper-parameters": best_config,
+ }
+ )
+
+ return (
+ search_time_list,
+ best_error_list,
+ error_list,
+ config_list,
+ logged_metric_list,
+ )
+
+
+def concat(X1, X2):
+ """concatenate two matrices vertically."""
+ if isinstance(X1, (DataFrame, Series)):
+ df = pd.concat([X1, X2], sort=False)
+ df.reset_index(drop=True, inplace=True)
+ if isinstance(X1, DataFrame):
+ cat_columns = X1.select_dtypes(include="category").columns
+ if len(cat_columns):
+ df[cat_columns] = df[cat_columns].astype("category")
+ return df
+ if issparse(X1):
+ return vstack((X1, X2))
+ else:
+ return np.concatenate([X1, X2])
+
+
+def add_time_idx_col(X):
+ unique_dates = X[TS_TIMESTAMP_COL].drop_duplicates().sort_values(ascending=True)
+ # assume no missing timestamps
+ freq = pd.infer_freq(unique_dates)
+ if freq == "MS":
+ X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year * 12 + X[TS_TIMESTAMP_COL].dt.month
+ elif freq == "Y":
+ X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year
+ else:
+ # using time frequency to generate all time stamps and then indexing for time_idx
+ # full_range = pd.date_range(X[TS_TIMESTAMP_COL].min(), X[TS_TIMESTAMP_COL].max(), freq=freq).to_list()
+ # X["time_idx"] = [full_range.index(time) for time in X[TS_TIMESTAMP_COL]]
+ # taking minimum difference in timestamp
+ timestamps = unique_dates.view("int64")
+ freq = int(timestamps.diff().mode())
+ X["time_idx"] = timestamps - timestamps.min() / freq
+ X["time_idx"] = X["time_idx"].astype("int")
+ return X
+
+
+class DataTransformer:
+ """Transform input training data."""
+
+ def fit_transform(self, X: Union[DataFrame, np.array], y, task):
+ """Fit transformer and process the input training data according to the task type.
+
+ Args:
+ X: A numpy array or a pandas dataframe of training data.
+ y: A numpy array or a pandas series of labels.
+ task: A string of the task type, e.g.,
+ 'classification', 'regression', 'ts_forecast', 'rank'.
+
+ Returns:
+ X: Processed numpy array or pandas dataframe of training data.
+ y: Processed numpy array or pandas series of labels.
+ """
+ if _is_nlp_task(task):
+ # if the mode is NLP, check the type of input, each column must be either string or
+ # ids (input ids, token type id, attention mask, etc.)
+ str_columns = []
+ for column in X.columns:
+ if isinstance(X[column].iloc[0], str):
+ str_columns.append(column)
+ if len(str_columns) > 0:
+ X[str_columns] = X[str_columns].astype("string")
+ self._str_columns = str_columns
+ elif isinstance(X, DataFrame):
+ X = X.copy()
+ n = X.shape[0]
+ cat_columns, num_columns, datetime_columns = [], [], []
+ drop = False
+ if task in TS_FORECAST:
+ X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
+ if task is TS_FORECASTPANEL:
+ if "time_idx" not in X:
+ X = add_time_idx_col(X)
+ ds_col = X.pop(TS_TIMESTAMP_COL)
+ if isinstance(y, Series):
+ y = y.rename(TS_VALUE_COL)
+ for column in X.columns:
+ # sklearn\utils\validation.py needs int/float values
+ if X[column].dtype.name in ("object", "category"):
+ if (
+ X[column].nunique() == 1
+ or X[column].nunique(dropna=True)
+ == n - X[column].isnull().sum()
+ ):
+ X.drop(columns=column, inplace=True)
+ drop = True
+ elif X[column].dtype.name == "category":
+ current_categories = X[column].cat.categories
+ if "__NAN__" not in current_categories:
+ X[column] = (
+ X[column]
+ .cat.add_categories("__NAN__")
+ .fillna("__NAN__")
+ )
+ cat_columns.append(column)
+ else:
+ X[column] = X[column].fillna("__NAN__")
+ cat_columns.append(column)
+ elif X[column].nunique(dropna=True) < 2:
+ X.drop(columns=column, inplace=True)
+ drop = True
+ else: # datetime or numeric
+ if X[column].dtype.name == "datetime64[ns]":
+ tmp_dt = X[column].dt
+ new_columns_dict = {
+ f"year_{column}": tmp_dt.year,
+ f"month_{column}": tmp_dt.month,
+ f"day_{column}": tmp_dt.day,
+ f"hour_{column}": tmp_dt.hour,
+ f"minute_{column}": tmp_dt.minute,
+ f"second_{column}": tmp_dt.second,
+ f"dayofweek_{column}": tmp_dt.dayofweek,
+ f"dayofyear_{column}": tmp_dt.dayofyear,
+ f"quarter_{column}": tmp_dt.quarter,
+ }
+ for key, value in new_columns_dict.items():
+ if (
+ key not in X.columns
+ and value.nunique(dropna=False) >= 2
+ ):
+ X[key] = value
+ num_columns.append(key)
+ X[column] = X[column].map(datetime.toordinal)
+ datetime_columns.append(column)
+ del tmp_dt
+ X[column] = X[column].fillna(np.nan)
+ num_columns.append(column)
+ X = X[cat_columns + num_columns]
+ if task in TS_FORECAST:
+ X.insert(0, TS_TIMESTAMP_COL, ds_col)
+ if cat_columns:
+ X[cat_columns] = X[cat_columns].astype("category")
+ if num_columns:
+ X_num = X[num_columns]
+ if np.issubdtype(X_num.columns.dtype, np.integer) and (
+ drop
+ or min(X_num.columns) != 0
+ or max(X_num.columns) != X_num.shape[1] - 1
+ ):
+ X_num.columns = range(X_num.shape[1])
+ drop = True
+ else:
+ drop = False
+ from sklearn.impute import SimpleImputer
+ from sklearn.compose import ColumnTransformer
+
+ self.transformer = ColumnTransformer(
+ [
+ (
+ "continuous",
+ SimpleImputer(missing_values=np.nan, strategy="median"),
+ X_num.columns,
+ )
+ ]
+ )
+ X[num_columns] = self.transformer.fit_transform(X_num)
+ self._cat_columns, self._num_columns, self._datetime_columns = (
+ cat_columns,
+ num_columns,
+ datetime_columns,
+ )
+ self._drop = drop
+ if (
+ task in CLASSIFICATION
+ or not pd.api.types.is_numeric_dtype(y)
+ and task not in NLG_TASKS
+ ):
+ if task != TOKENCLASSIFICATION:
+ from sklearn.preprocessing import LabelEncoder
+
+ self.label_transformer = LabelEncoder()
+ else:
+ from flaml.automl.nlp.utils import LabelEncoderforTokenClassification
+
+ self.label_transformer = LabelEncoderforTokenClassification()
+ y = self.label_transformer.fit_transform(y)
+
+ else:
+ self.label_transformer = None
+ self._task = task
+ return X, y
+
+ def transform(self, X: Union[DataFrame, np.array]):
+ """Process data using fit transformer.
+
+ Args:
+ X: A numpy array or a pandas dataframe of training data.
+
+ Returns:
+ X: Processed numpy array or pandas dataframe of training data.
+ """
+ X = X.copy()
+
+ if _is_nlp_task(self._task):
+ # if the mode is NLP, check the type of input, each column must be either string or
+ # ids (input ids, token type id, attention mask, etc.)
+ if len(self._str_columns) > 0:
+ X[self._str_columns] = X[self._str_columns].astype("string")
+ elif isinstance(X, DataFrame):
+ cat_columns, num_columns, datetime_columns = (
+ self._cat_columns,
+ self._num_columns,
+ self._datetime_columns,
+ )
+ if self._task in TS_FORECAST:
+ X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
+ ds_col = X.pop(TS_TIMESTAMP_COL)
+ for column in datetime_columns:
+ tmp_dt = X[column].dt
+ new_columns_dict = {
+ f"year_{column}": tmp_dt.year,
+ f"month_{column}": tmp_dt.month,
+ f"day_{column}": tmp_dt.day,
+ f"hour_{column}": tmp_dt.hour,
+ f"minute_{column}": tmp_dt.minute,
+ f"second_{column}": tmp_dt.second,
+ f"dayofweek_{column}": tmp_dt.dayofweek,
+ f"dayofyear_{column}": tmp_dt.dayofyear,
+ f"quarter_{column}": tmp_dt.quarter,
+ }
+ for new_col_name, new_col_value in new_columns_dict.items():
+ if new_col_name not in X.columns and new_col_name in num_columns:
+ X[new_col_name] = new_col_value
+ X[column] = X[column].map(datetime.toordinal)
+ del tmp_dt
+ X = X[cat_columns + num_columns].copy()
+ if self._task in TS_FORECAST:
+ X.insert(0, TS_TIMESTAMP_COL, ds_col)
+ for column in cat_columns:
+ if X[column].dtype.name == "object":
+ X[column] = X[column].fillna("__NAN__")
+ elif X[column].dtype.name == "category":
+ current_categories = X[column].cat.categories
+ if "__NAN__" not in current_categories:
+ X[column] = (
+ X[column].cat.add_categories("__NAN__").fillna("__NAN__")
+ )
+ if cat_columns:
+ X[cat_columns] = X[cat_columns].astype("category")
+ if num_columns:
+ X_num = X[num_columns].fillna(np.nan)
+ if self._drop:
+ X_num.columns = range(X_num.shape[1])
+ X[num_columns] = self.transformer.transform(X_num)
+ return X
+
+
+def group_counts(groups):
+ _, i, c = np.unique(groups, return_counts=True, return_index=True)
+ return c[np.argsort(i)]
diff --git a/flaml/automl/ml.py b/flaml/automl/ml.py
new file mode 100644
index 000000000..6285bc29c
--- /dev/null
+++ b/flaml/automl/ml.py
@@ -0,0 +1,749 @@
+# !
+# * Copyright (c) FLAML authors. All rights reserved.
+# * Licensed under the MIT License. See LICENSE file in the
+# * project root for license information.
+import time
+import numpy as np
+import pandas as pd
+from sklearn.metrics import (
+ mean_squared_error,
+ r2_score,
+ roc_auc_score,
+ accuracy_score,
+ mean_absolute_error,
+ log_loss,
+ average_precision_score,
+ f1_score,
+ mean_absolute_percentage_error,
+ ndcg_score,
+)
+from sklearn.model_selection import RepeatedStratifiedKFold, GroupKFold, TimeSeriesSplit
+from flaml.automl.model import (
+ XGBoostSklearnEstimator,
+ XGBoost_TS,
+ XGBoostLimitDepthEstimator,
+ XGBoostLimitDepth_TS,
+ RandomForestEstimator,
+ RF_TS,
+ LGBMEstimator,
+ LGBM_TS,
+ LRL1Classifier,
+ LRL2Classifier,
+ CatBoostEstimator,
+ ExtraTreesEstimator,
+ ExtraTrees_TS,
+ KNeighborsEstimator,
+ Prophet,
+ ARIMA,
+ SARIMAX,
+ TransformersEstimator,
+ TemporalFusionTransformerEstimator,
+ TransformersEstimatorModelSelection,
+)
+from flaml.automl.data import CLASSIFICATION, group_counts, TS_FORECAST
+import logging
+
+logger = logging.getLogger(__name__)
+
+sklearn_metric_name_set = {
+ "r2",
+ "rmse",
+ "mae",
+ "mse",
+ "accuracy",
+ "roc_auc",
+ "roc_auc_ovr",
+ "roc_auc_ovo",
+ "roc_auc_weighted",
+ "roc_auc_ovr_weighted",
+ "roc_auc_ovo_weighted",
+ "log_loss",
+ "mape",
+ "f1",
+ "ap",
+ "ndcg",
+ "micro_f1",
+ "macro_f1",
+}
+huggingface_metric_to_mode = {
+ "accuracy": "max",
+ "bertscore": "max",
+ "bleu": "max",
+ "bleurt": "max",
+ "cer": "min",
+ "chrf": "min",
+ "code_eval": "max",
+ "comet": "max",
+ "competition_math": "max",
+ "coval": "max",
+ "cuad": "max",
+ "f1": "max",
+ "gleu": "max",
+ "google_bleu": "max",
+ "matthews_correlation": "max",
+ "meteor": "max",
+ "pearsonr": "max",
+ "precision": "max",
+ "recall": "max",
+ "rouge": "max",
+ "sacrebleu": "max",
+ "sari": "max",
+ "seqeval": "max",
+ "spearmanr": "max",
+ "ter": "min",
+ "wer": "min",
+}
+huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}
+
+
+def get_estimator_class(task, estimator_name):
+ # when adding a new learner, need to add an elif branch
+ if "xgboost" == estimator_name:
+ estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator
+ elif "xgb_limitdepth" == estimator_name:
+ estimator_class = (
+ XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator
+ )
+ elif "rf" == estimator_name:
+ estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator
+ elif "lgbm" == estimator_name:
+ estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator
+ elif "lrl1" == estimator_name:
+ estimator_class = LRL1Classifier
+ elif "lrl2" == estimator_name:
+ estimator_class = LRL2Classifier
+ elif "catboost" == estimator_name:
+ estimator_class = CatBoostEstimator
+ elif "extra_tree" == estimator_name:
+ estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator
+ elif "kneighbor" == estimator_name:
+ estimator_class = KNeighborsEstimator
+ elif "prophet" in estimator_name:
+ estimator_class = Prophet
+ elif estimator_name == "arima":
+ estimator_class = ARIMA
+ elif estimator_name == "sarimax":
+ estimator_class = SARIMAX
+ elif estimator_name == "transformer":
+ estimator_class = TransformersEstimator
+ elif estimator_name == "tft":
+ estimator_class = TemporalFusionTransformerEstimator
+ elif estimator_name == "transformer_ms":
+ estimator_class = TransformersEstimatorModelSelection
+ else:
+ raise ValueError(
+ estimator_name + " is not a built-in learner. "
+ "Please use AutoML.add_learner() to add a customized learner."
+ )
+ return estimator_class
+
+
+def metric_loss_score(
+ metric_name,
+ y_processed_predict,
+ y_processed_true,
+ labels=None,
+ sample_weight=None,
+ groups=None,
+):
+ # y_processed_predict and y_processed_true are processed id labels if the original were the token labels
+ if is_in_sklearn_metric_name_set(metric_name):
+ return sklearn_metric_loss_score(
+ metric_name,
+ y_processed_predict,
+ y_processed_true,
+ labels,
+ sample_weight,
+ groups,
+ )
+ else:
+ try:
+ import datasets
+
+ datasets_metric_name = huggingface_submetric_to_metric.get(
+ metric_name, metric_name.split(":")[0]
+ )
+ metric = datasets.load_metric(datasets_metric_name)
+ metric_mode = huggingface_metric_to_mode[datasets_metric_name]
+
+ if metric_name.startswith("seqeval"):
+ y_processed_true = [
+ [labels[tr] for tr in each_list] for each_list in y_processed_true
+ ]
+ elif metric in ("pearsonr", "spearmanr"):
+ y_processed_true = (
+ y_processed_true.to_list()
+ if isinstance(y_processed_true, pd.Series)
+ else list(y_processed_true)
+ )
+ score_dict = metric.compute(
+ predictions=y_processed_predict, references=y_processed_true
+ )
+ if "rouge" in metric_name:
+ score = score_dict[metric_name].mid.fmeasure
+ elif metric_name.startswith("seqeval"):
+ metric_submetric_names = metric_name.split(":")
+ score = score_dict[
+ metric_submetric_names[1]
+ if len(metric_submetric_names) > 1
+ else "overall_accuracy"
+ ]
+ else:
+ score = score_dict[metric_name]
+ except ImportError:
+ raise ValueError(
+ metric_name
+ + " is not an built-in sklearn metric and nlp is not installed. "
+ "Currently built-in sklearn metrics are: "
+ "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"
+ "log_loss, mape, f1, micro_f1, macro_f1, ap. "
+ "If the metric is an nlp metric, please pip install flaml[nlp] ",
+ "or pass a customized metric function to AutoML.fit(metric=func)",
+ )
+ # If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)
+ # ask the user to provide a custom metric
+ except FileNotFoundError:
+ raise ValueError(
+ metric_name + " is neither an sklearn metric nor a huggingface metric. "
+ "Currently built-in sklearn metrics are: "
+ "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"
+ "log_loss, mape, f1, micro_f1, macro_f1, ap. "
+ "Currently built-in huggingface metrics are: "
+ + ", ".join(huggingface_metric_to_mode.keys())
+ + ". Please pass a customized metric function to AutoML.fit(metric=func)"
+ )
+ if metric_mode == "max":
+ return 1 - score
+ else:
+ return score
+
+
+def is_in_sklearn_metric_name_set(metric_name):
+ return metric_name.startswith("ndcg") or metric_name in sklearn_metric_name_set
+
+
+def is_min_metric(metric_name):
+ return (
+ metric_name in ["rmse", "mae", "mse", "log_loss", "mape"]
+ or huggingface_metric_to_mode.get(metric_name, None) == "min"
+ )
+
+
+def sklearn_metric_loss_score(
+ metric_name,
+ y_predict,
+ y_true,
+ labels=None,
+ sample_weight=None,
+ groups=None,
+):
+ """Loss using the specified metric.
+
+ Args:
+ metric_name: A string of the metric name, one of
+ 'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'roc_auc_ovr',
+ 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted',
+ 'log_loss', 'mape', 'f1', 'ap', 'ndcg', 'micro_f1', 'macro_f1'.
+ y_predict: A 1d or 2d numpy array of the predictions which can be
+ used to calculate the metric. E.g., 2d for log_loss and 1d
+ for others.
+ y_true: A 1d numpy array of the true labels.
+ labels: A list or an array of the unique labels.
+ sample_weight: A 1d numpy array of the sample weight.
+ groups: A 1d numpy array of the group labels.
+
+ Returns:
+ score: A float number of the loss, the lower the better.
+ """
+ metric_name = metric_name.lower()
+
+ if "r2" == metric_name:
+ score = 1.0 - r2_score(y_true, y_predict, sample_weight=sample_weight)
+ elif metric_name == "rmse":
+ score = np.sqrt(
+ mean_squared_error(y_true, y_predict, sample_weight=sample_weight)
+ )
+ elif metric_name == "mae":
+ score = mean_absolute_error(y_true, y_predict, sample_weight=sample_weight)
+ elif metric_name == "mse":
+ score = mean_squared_error(y_true, y_predict, sample_weight=sample_weight)
+ elif metric_name == "accuracy":
+ score = 1.0 - accuracy_score(y_true, y_predict, sample_weight=sample_weight)
+ elif metric_name == "roc_auc":
+ score = 1.0 - roc_auc_score(y_true, y_predict, sample_weight=sample_weight)
+ elif metric_name == "roc_auc_ovr":
+ score = 1.0 - roc_auc_score(
+ y_true, y_predict, sample_weight=sample_weight, multi_class="ovr"
+ )
+ elif metric_name == "roc_auc_ovo":
+ score = 1.0 - roc_auc_score(
+ y_true, y_predict, sample_weight=sample_weight, multi_class="ovo"
+ )
+ elif metric_name == "roc_auc_weighted":
+ score = 1.0 - roc_auc_score(
+ y_true, y_predict, sample_weight=sample_weight, average="weighted"
+ )
+ elif metric_name == "roc_auc_ovo_weighted":
+ score = 1.0 - roc_auc_score(
+ y_true,
+ y_predict,
+ sample_weight=sample_weight,
+ average="weighted",
+ multi_class="ovo",
+ )
+ elif metric_name == "roc_auc_ovr_weighted":
+ score = 1.0 - roc_auc_score(
+ y_true,
+ y_predict,
+ sample_weight=sample_weight,
+ average="weighted",
+ multi_class="ovr",
+ )
+ elif "log_loss" == metric_name:
+ score = log_loss(y_true, y_predict, labels=labels, sample_weight=sample_weight)
+ elif "mape" == metric_name:
+ try:
+ score = mean_absolute_percentage_error(y_true, y_predict)
+ except ValueError:
+ return np.inf
+ elif "micro_f1" == metric_name:
+ score = 1 - f1_score(
+ y_true, y_predict, sample_weight=sample_weight, average="micro"
+ )
+ elif "macro_f1" == metric_name:
+ score = 1 - f1_score(
+ y_true, y_predict, sample_weight=sample_weight, average="macro"
+ )
+ elif "f1" == metric_name:
+ score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight)
+ elif "ap" == metric_name:
+ score = 1 - average_precision_score(
+ y_true, y_predict, sample_weight=sample_weight
+ )
+ elif "ndcg" in metric_name:
+ if "@" in metric_name:
+ k = int(metric_name.split("@", 1)[-1])
+ counts = group_counts(groups)
+ score = 0
+ psum = 0
+ for c in counts:
+ score -= ndcg_score(
+ np.asarray([y_true[psum : psum + c]]),
+ np.asarray([y_predict[psum : psum + c]]),
+ k=k,
+ )
+ psum += c
+ score /= len(counts)
+ score += 1
+ else:
+ score = 1 - ndcg_score([y_true], [y_predict])
+ return score
+
+
+def get_y_pred(estimator, X, eval_metric, obj):
+ if eval_metric in ["roc_auc", "ap", "roc_auc_weighted"] and "binary" in obj:
+ y_pred_classes = estimator.predict_proba(X)
+ y_pred = y_pred_classes[:, 1] if y_pred_classes.ndim > 1 else y_pred_classes
+ elif eval_metric in [
+ "log_loss",
+ "roc_auc",
+ "roc_auc_ovr",
+ "roc_auc_ovo",
+ "roc_auc_ovo_weighted",
+ "roc_auc_ovr_weighted",
+ ]:
+ y_pred = estimator.predict_proba(X)
+ else:
+ y_pred = estimator.predict(X)
+ return y_pred
+
+
+def _eval_estimator(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ obj,
+ labels=None,
+ log_training_metric=False,
+ fit_kwargs={},
+):
+ if isinstance(eval_metric, str):
+ pred_start = time.time()
+ val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)
+ pred_time = (time.time() - pred_start) / X_val.shape[0]
+
+ val_loss = metric_loss_score(
+ eval_metric,
+ y_processed_predict=val_pred_y,
+ y_processed_true=y_val,
+ labels=labels,
+ sample_weight=weight_val,
+ groups=groups_val,
+ )
+ metric_for_logging = {"pred_time": pred_time}
+ if log_training_metric:
+ train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj)
+ metric_for_logging["train_loss"] = metric_loss_score(
+ eval_metric,
+ train_pred_y,
+ y_train,
+ labels,
+ fit_kwargs.get("sample_weight"),
+ fit_kwargs.get("groups"),
+ )
+ else: # customized metric function
+ val_loss, metric_for_logging = eval_metric(
+ X_val,
+ y_val,
+ estimator,
+ labels,
+ X_train,
+ y_train,
+ weight_val,
+ fit_kwargs.get("sample_weight"),
+ config,
+ groups_val,
+ fit_kwargs.get("groups"),
+ )
+ pred_time = metric_for_logging.get("pred_time", 0)
+ val_pred_y = None
+ # eval_metric may return val_pred_y but not necessarily. Setting None for now.
+ return val_loss, metric_for_logging, pred_time, val_pred_y
+
+
+def get_val_loss(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ obj,
+ labels=None,
+ budget=None,
+ log_training_metric=False,
+ fit_kwargs={},
+ free_mem_ratio=0,
+):
+
+ start = time.time()
+ # if groups_val is not None:
+ # fit_kwargs['groups_val'] = groups_val
+ # fit_kwargs['X_val'] = X_val
+ # fit_kwargs['y_val'] = y_val
+ estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)
+ val_loss, metric_for_logging, pred_time, _ = _eval_estimator(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ obj,
+ labels,
+ log_training_metric,
+ fit_kwargs,
+ )
+ if hasattr(estimator, "intermediate_results"):
+ metric_for_logging["intermediate_results"] = estimator.intermediate_results
+ train_time = time.time() - start
+ return val_loss, metric_for_logging, train_time, pred_time
+
+
+def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):
+ metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)
+ metrics_to_log = None
+ for single_fold in log_metrics_folds:
+ if metrics_to_log is None:
+ metrics_to_log = single_fold
+ elif isinstance(metrics_to_log, dict):
+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
+ else:
+ metrics_to_log += single_fold
+ if metrics_to_log:
+ n = len(val_loss_folds)
+ metrics_to_log = (
+ {k: v / n for k, v in metrics_to_log.items()}
+ if isinstance(metrics_to_log, dict)
+ else metrics_to_log / n
+ )
+ return metric_to_minimize, metrics_to_log
+
+
+def evaluate_model_CV(
+ config,
+ estimator,
+ X_train_all,
+ y_train_all,
+ budget,
+ kf,
+ task,
+ eval_metric,
+ best_val_loss,
+ cv_score_agg_func=None,
+ log_training_metric=False,
+ fit_kwargs={},
+ free_mem_ratio=0,
+):
+ if cv_score_agg_func is None:
+ cv_score_agg_func = default_cv_score_agg_func
+ start_time = time.time()
+ val_loss_folds = []
+ log_metric_folds = []
+ metric = None
+ train_time = pred_time = 0
+ total_fold_num = 0
+ n = kf.get_n_splits()
+ X_train_split, y_train_split = X_train_all, y_train_all
+ if task in CLASSIFICATION:
+ labels = np.unique(y_train_all)
+ else:
+ labels = fit_kwargs.get(
+ "label_list"
+ ) # pass the label list on to compute the evaluation metric
+ groups = None
+ shuffle = getattr(kf, "shuffle", task not in TS_FORECAST)
+ if isinstance(kf, RepeatedStratifiedKFold):
+ kf = kf.split(X_train_split, y_train_split)
+ elif isinstance(kf, GroupKFold):
+ groups = kf.groups
+ kf = kf.split(X_train_split, y_train_split, groups)
+ shuffle = False
+ elif isinstance(kf, TimeSeriesSplit):
+ kf = kf.split(X_train_split, y_train_split)
+ else:
+ kf = kf.split(X_train_split)
+ rng = np.random.RandomState(2020)
+ budget_per_train = budget and budget / n
+ if "sample_weight" in fit_kwargs:
+ weight = fit_kwargs["sample_weight"]
+ weight_val = None
+ else:
+ weight = weight_val = None
+ for train_index, val_index in kf:
+ if shuffle:
+ train_index = rng.permutation(train_index)
+ if isinstance(X_train_all, pd.DataFrame):
+ X_train = X_train_split.iloc[train_index]
+ X_val = X_train_split.iloc[val_index]
+ else:
+ X_train, X_val = X_train_split[train_index], X_train_split[val_index]
+ y_train, y_val = y_train_split[train_index], y_train_split[val_index]
+ estimator.cleanup()
+ if weight is not None:
+ fit_kwargs["sample_weight"], weight_val = (
+ weight[train_index],
+ weight[val_index],
+ )
+ if groups is not None:
+ fit_kwargs["groups"] = groups[train_index]
+ groups_val = groups[val_index]
+ else:
+ groups_val = None
+ val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ task,
+ labels,
+ budget_per_train,
+ log_training_metric=log_training_metric,
+ fit_kwargs=fit_kwargs,
+ free_mem_ratio=free_mem_ratio,
+ )
+ if isinstance(metric_i, dict) and "intermediate_results" in metric_i.keys():
+ del metric_i["intermediate_results"]
+ if weight is not None:
+ fit_kwargs["sample_weight"] = weight
+ total_fold_num += 1
+ val_loss_folds.append(val_loss_i)
+ log_metric_folds.append(metric_i)
+ train_time += train_time_i
+ pred_time += pred_time_i
+ if budget and time.time() - start_time >= budget:
+ break
+ val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)
+ n = total_fold_num
+ pred_time /= n
+ return val_loss, metric, train_time, pred_time
+
+
+def compute_estimator(
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ budget,
+ kf,
+ config_dic,
+ task,
+ estimator_name,
+ eval_method,
+ eval_metric,
+ best_val_loss=np.Inf,
+ n_jobs=1,
+ estimator_class=None,
+ cv_score_agg_func=None,
+ log_training_metric=False,
+ fit_kwargs={},
+ free_mem_ratio=0,
+):
+ estimator_class = estimator_class or get_estimator_class(task, estimator_name)
+ estimator = estimator_class(
+ **config_dic,
+ task=task,
+ n_jobs=n_jobs,
+ )
+
+ if isinstance(estimator, TransformersEstimator):
+ # TODO: move the partial function to nlp
+ fit_kwargs["metric"] = eval_metric
+ fit_kwargs["X_val"] = X_val
+ fit_kwargs["y_val"] = y_val
+
+ if "holdout" == eval_method:
+ val_loss, metric_for_logging, train_time, pred_time = get_val_loss(
+ config_dic,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ task,
+ labels=fit_kwargs.get(
+ "label_list"
+ ), # pass the label list on to compute the evaluation metric
+ budget=budget,
+ log_training_metric=log_training_metric,
+ fit_kwargs=fit_kwargs,
+ free_mem_ratio=0,
+ )
+ else:
+ val_loss, metric_for_logging, train_time, pred_time = evaluate_model_CV(
+ config_dic,
+ estimator,
+ X_train,
+ y_train,
+ budget,
+ kf,
+ task,
+ eval_metric,
+ best_val_loss,
+ cv_score_agg_func,
+ log_training_metric=log_training_metric,
+ fit_kwargs=fit_kwargs,
+ free_mem_ratio=0,
+ )
+
+ if isinstance(estimator, TransformersEstimator):
+ del fit_kwargs["metric"], fit_kwargs["X_val"], fit_kwargs["y_val"]
+
+ return estimator, val_loss, metric_for_logging, train_time, pred_time
+
+
+def train_estimator(
+ config_dic,
+ X_train,
+ y_train,
+ task,
+ estimator_name,
+ n_jobs=1,
+ estimator_class=None,
+ budget=None,
+ fit_kwargs={},
+ eval_metric=None,
+ free_mem_ratio=0,
+):
+ start_time = time.time()
+ estimator_class = estimator_class or get_estimator_class(task, estimator_name)
+ estimator = estimator_class(
+ **config_dic,
+ task=task,
+ n_jobs=n_jobs,
+ )
+ if isinstance(estimator, TransformersEstimator):
+ fit_kwargs["metric"] = eval_metric
+
+ if X_train is not None:
+ train_time = estimator.fit(
+ X_train, y_train, budget, free_mem_ratio, **fit_kwargs
+ )
+ else:
+ estimator = estimator.estimator_class(**estimator.params)
+ train_time = time.time() - start_time
+ return estimator, train_time
+
+
+def get_classification_objective(num_labels: int) -> str:
+ if num_labels == 2:
+ objective_name = "binary"
+ else:
+ objective_name = "multiclass"
+ return objective_name
+
+
+def norm_confusion_matrix(y_true, y_pred):
+ """normalized confusion matrix.
+
+ Args:
+ estimator: A multi-class classification estimator.
+ y_true: A numpy array or a pandas series of true labels.
+ y_pred: A numpy array or a pandas series of predicted labels.
+
+ Returns:
+ A normalized confusion matrix.
+ """
+ from sklearn.metrics import confusion_matrix
+
+ conf_mat = confusion_matrix(y_true, y_pred)
+ norm_conf_mat = conf_mat.astype("float") / conf_mat.sum(axis=1)[:, np.newaxis]
+ return norm_conf_mat
+
+
+def multi_class_curves(y_true, y_pred_proba, curve_func):
+ """Binarize the data for multi-class tasks and produce ROC or precision-recall curves.
+
+ Args:
+ y_true: A numpy array or a pandas series of true labels.
+ y_pred_proba: A numpy array or a pandas dataframe of predicted probabilites.
+ curve_func: A function to produce a curve (e.g., roc_curve or precision_recall_curve).
+
+ Returns:
+ A tuple of two dictionaries with the same set of keys (class indices).
+ The first dictionary curve_x stores the x coordinates of each curve, e.g.,
+ curve_x[0] is an 1D array of the x coordinates of class 0.
+ The second dictionary curve_y stores the y coordinates of each curve, e.g.,
+ curve_y[0] is an 1D array of the y coordinates of class 0.
+ """
+ from sklearn.preprocessing import label_binarize
+
+ classes = np.unique(y_true)
+ y_true_binary = label_binarize(y_true, classes=classes)
+
+ curve_x, curve_y = {}, {}
+ for i in range(len(classes)):
+ curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])
+ return curve_x, curve_y
diff --git a/flaml/automl/model.py b/flaml/automl/model.py
new file mode 100644
index 000000000..bccad6470
--- /dev/null
+++ b/flaml/automl/model.py
@@ -0,0 +1,2393 @@
+# !
+# * Copyright (c) FLAML authors. All rights reserved.
+# * Licensed under the MIT License. See LICENSE file in the
+# * project root for license information.
+from contextlib import contextmanager
+from functools import partial
+import signal
+import os
+from typing import Callable, List
+import numpy as np
+import time
+from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
+from sklearn.ensemble import ExtraTreesRegressor, ExtraTreesClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.dummy import DummyClassifier, DummyRegressor
+from scipy.sparse import issparse
+import logging
+import shutil
+from pandas import DataFrame, Series, to_datetime
+import sys
+import math
+from flaml import tune
+from flaml.automl.data import (
+ group_counts,
+ CLASSIFICATION,
+ add_time_idx_col,
+ TS_FORECASTREGRESSION,
+ TS_TIMESTAMP_COL,
+ TS_VALUE_COL,
+ SEQCLASSIFICATION,
+ SEQREGRESSION,
+ TOKENCLASSIFICATION,
+ SUMMARIZATION,
+ NLG_TASKS,
+)
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+try:
+ import resource
+except ImportError:
+ resource = None
+
+logger = logging.getLogger("flaml.automl")
+# FREE_MEM_RATIO = 0.2
+
+
+def TimeoutHandler(sig, frame):
+ raise TimeoutError(sig, frame)
+
+
+@contextmanager
+def limit_resource(memory_limit, time_limit):
+ if memory_limit > 0:
+ soft, hard = resource.getrlimit(resource.RLIMIT_AS)
+ if soft < 0 and (hard < 0 or memory_limit <= hard) or memory_limit < soft:
+ try:
+ resource.setrlimit(resource.RLIMIT_AS, (int(memory_limit), hard))
+ except ValueError:
+ # According to https://bugs.python.org/issue40518, it's a mac-specific error.
+ pass
+ main_thread = False
+ if time_limit is not None:
+ try:
+ signal.signal(signal.SIGALRM, TimeoutHandler)
+ signal.alarm(int(time_limit) or 1)
+ main_thread = True
+ except ValueError:
+ pass
+ try:
+ yield
+ finally:
+ if main_thread:
+ signal.alarm(0)
+ if memory_limit > 0:
+ resource.setrlimit(resource.RLIMIT_AS, (soft, hard))
+
+
+class BaseEstimator:
+ """The abstract class for all learners.
+
+ Typical examples:
+ * XGBoostEstimator: for regression.
+ * XGBoostSklearnEstimator: for classification.
+ * LGBMEstimator, RandomForestEstimator, LRL1Classifier, LRL2Classifier:
+ for both regression and classification.
+ """
+
+ def __init__(self, task="binary", **config):
+ """Constructor.
+
+ Args:
+ task: A string of the task type, one of
+ 'binary', 'multiclass', 'regression', 'rank', 'seq-classification',
+ 'seq-regression', 'token-classification', 'multichoice-classification',
+ 'summarization', 'ts_forecast', 'ts_forecast_classification'.
+ config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.
+ n_jobs is the number of parallel threads.
+ """
+ self._task = task
+ self.params = self.config2params(config)
+ self.estimator_class = self._model = None
+ if "_estimator_type" in config:
+ self._estimator_type = self.params.pop("_estimator_type")
+ else:
+ self._estimator_type = (
+ "classifier" if task in CLASSIFICATION else "regressor"
+ )
+
+ def get_params(self, deep=False):
+ params = self.params.copy()
+ params["task"] = self._task
+ if hasattr(self, "_estimator_type"):
+ params["_estimator_type"] = self._estimator_type
+ return params
+
+ @property
+ def classes_(self):
+ return self._model.classes_
+
+ @property
+ def n_features_in_(self):
+ return self._model.n_features_in_
+
+ @property
+ def model(self):
+ """Trained model after fit() is called, or None before fit() is called."""
+ return self._model
+
+ @property
+ def estimator(self):
+ """Trained model after fit() is called, or None before fit() is called."""
+ return self._model
+
+ @property
+ def feature_names_in_(self):
+ """
+ if self._model has attribute feature_names_in_, return it.
+ otherwise, if self._model has attribute feature_name_, return it.
+ otherwise, if self._model has attribute feature_names, return it.
+ otherwise, if self._model has method get_booster, return the feature names.
+ otherwise, return None.
+ """
+ if hasattr(self._model, "feature_names_in_"): # for sklearn, xgboost>=1.6
+ return self._model.feature_names_in_
+ if hasattr(self._model, "feature_name_"): # for lightgbm
+ return self._model.feature_name_
+ if hasattr(self._model, "feature_names"): # for XGBoostEstimator
+ return self._model.feature_names
+ if hasattr(self._model, "get_booster"):
+ # get feature names for xgboost<1.6
+ # https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.Booster.feature_names
+ booster = self._model.get_booster()
+ return booster.feature_names
+ return None
+
+ @property
+ def feature_importances_(self):
+ """
+ if self._model has attribute feature_importances_, return it.
+ otherwise, if self._model has attribute coef_, return it.
+ otherwise, return None.
+ """
+ if hasattr(self._model, "feature_importances_"):
+ # for sklearn, lightgbm, catboost, xgboost
+ return self._model.feature_importances_
+ elif hasattr(self._model, "coef_"): # for linear models
+ return self._model.coef_
+ else:
+ return None
+
+ def _preprocess(self, X):
+ return X
+
+ def _fit(self, X_train, y_train, **kwargs):
+
+ current_time = time.time()
+ if "groups" in kwargs:
+ kwargs = kwargs.copy()
+ groups = kwargs.pop("groups")
+ if self._task == "rank":
+ kwargs["group"] = group_counts(groups)
+ # groups_val = kwargs.get('groups_val')
+ # if groups_val is not None:
+ # kwargs['eval_group'] = [group_counts(groups_val)]
+ # kwargs['eval_set'] = [
+ # (kwargs['X_val'], kwargs['y_val'])]
+ # kwargs['verbose'] = False
+ # del kwargs['groups_val'], kwargs['X_val'], kwargs['y_val']
+ X_train = self._preprocess(X_train)
+ model = self.estimator_class(**self.params)
+ if logger.level == logging.DEBUG:
+ # xgboost 1.6 doesn't display all the params in the model str
+ logger.debug(f"flaml.model - {model} fit started with params {self.params}")
+ model.fit(X_train, y_train, **kwargs)
+ if logger.level == logging.DEBUG:
+ logger.debug(f"flaml.model - {model} fit finished")
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ """Train the model from given training data.
+
+ Args:
+ X_train: A numpy array or a dataframe of training data in shape n*m.
+ y_train: A numpy array or a series of labels in shape n*1.
+ budget: A float of the time budget in seconds.
+ free_mem_ratio: A float between 0 and 1 for the free memory ratio to keep during training.
+
+ Returns:
+ train_time: A float of the training time in seconds.
+ """
+ if (
+ getattr(self, "limit_resource", None)
+ and resource is not None
+ and (budget is not None or psutil is not None)
+ ):
+ start_time = time.time()
+ mem = psutil.virtual_memory() if psutil is not None else None
+ try:
+ with limit_resource(
+ mem.available * (1 - free_mem_ratio)
+ + psutil.Process(os.getpid()).memory_info().rss
+ if mem is not None
+ else -1,
+ budget,
+ ):
+ train_time = self._fit(X_train, y_train, **kwargs)
+ except (MemoryError, TimeoutError) as e:
+ logger.warning(f"{e.__class__} {e}")
+ if self._task in CLASSIFICATION:
+ model = DummyClassifier()
+ else:
+ model = DummyRegressor()
+ X_train = self._preprocess(X_train)
+ model.fit(X_train, y_train)
+ self._model = model
+ train_time = time.time() - start_time
+ else:
+ train_time = self._fit(X_train, y_train, **kwargs)
+ return train_time
+
+ def predict(self, X, **kwargs):
+ """Predict label from features.
+
+ Args:
+ X: A numpy array or a dataframe of featurized instances, shape n*m.
+
+ Returns:
+ A numpy array of shape n*1.
+ Each element is the label for a instance.
+ """
+ if self._model is not None:
+ X = self._preprocess(X)
+ return self._model.predict(X, **kwargs)
+ else:
+ logger.warning(
+ "Estimator is not fit yet. Please run fit() before predict()."
+ )
+ return np.ones(X.shape[0])
+
+ def predict_proba(self, X, **kwargs):
+ """Predict the probability of each class from features.
+
+ Only works for classification problems
+
+ Args:
+ X: A numpy array of featurized instances, shape n*m.
+
+ Returns:
+ A numpy array of shape n*c. c is the # classes.
+ Each element at (i,j) is the probability for instance i to be in
+ class j.
+ """
+ assert self._task in CLASSIFICATION, "predict_proba() only for classification."
+
+ X = self._preprocess(X)
+ return self._model.predict_proba(X, **kwargs)
+
+ def score(self, X_val: DataFrame, y_val: Series, **kwargs):
+ """Report the evaluation score of a trained estimator.
+
+
+ Args:
+ X_val: A pandas dataframe of the validation input data.
+ y_val: A pandas series of the validation label.
+ kwargs: keyword argument of the evaluation function, for example:
+ - metric: A string of the metric name or a function
+ e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',
+ 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',
+ 'mape'. Default is 'auto'.
+ If metric is given, the score will report the user specified metric.
+ If metric is not given, the metric is set to accuracy for classification and r2
+ for regression.
+ You can also pass a customized metric function, for examples on how to pass a
+ customized metric function, please check
+ [test/nlp/test_autohf_custom_metric.py](https://github.com/microsoft/FLAML/blob/main/test/nlp/test_autohf_custom_metric.py) and
+ [test/automl/test_multiclass.py](https://github.com/microsoft/FLAML/blob/main/test/automl/test_multiclass.py).
+
+ Returns:
+ The evaluation score on the validation dataset.
+ """
+ from .ml import metric_loss_score
+ from .ml import is_min_metric
+
+ if self._model is not None:
+ if self._task == "rank":
+ raise NotImplementedError(
+ "AutoML.score() is not implemented for ranking"
+ )
+ else:
+ X_val = self._preprocess(X_val)
+ metric = kwargs.pop("metric", None)
+ if metric:
+ y_pred = self.predict(X_val, **kwargs)
+ if is_min_metric(metric):
+ return metric_loss_score(metric, y_pred, y_val)
+ else:
+ return 1.0 - metric_loss_score(metric, y_pred, y_val)
+ else:
+ return self._model.score(X_val, y_val, **kwargs)
+ else:
+ logger.warning(
+ "Estimator is not fit yet. Please run fit() before predict()."
+ )
+ return 0.0
+
+ def cleanup(self):
+ del self._model
+ self._model = None
+
+ @classmethod
+ def search_space(cls, data_size, task, **params):
+ """[required method] search space.
+
+ Args:
+ data_size: A tuple of two integers, number of rows and columns.
+ task: A str of the task type, e.g., "binary", "multiclass", "regression".
+
+ Returns:
+ A dictionary of the search space.
+ Each key is the name of a hyperparameter, and value is a dict with
+ its domain (required) and low_cost_init_value, init_value,
+ cat_hp_cost (if applicable).
+ e.g., ```{'domain': tune.randint(lower=1, upper=10), 'init_value': 1}```.
+ """
+ return {}
+
+ @classmethod
+ def size(cls, config: dict) -> float:
+ """[optional method] memory size of the estimator in bytes.
+
+ Args:
+ config: A dict of the hyperparameter config.
+
+ Returns:
+ A float of the memory size required by the estimator to train the
+ given config.
+ """
+ return 1.0
+
+ @classmethod
+ def cost_relative2lgbm(cls) -> float:
+ """[optional method] relative cost compared to lightgbm."""
+ return 1.0
+
+ @classmethod
+ def init(cls):
+ """[optional method] initialize the class."""
+ pass
+
+ def config2params(self, config: dict) -> dict:
+ """[optional method] config dict to params dict
+
+ Args:
+ config: A dict of the hyperparameter config.
+
+ Returns:
+ A dict that will be passed to self.estimator_class's constructor.
+ """
+ params = config.copy()
+ if "FLAML_sample_size" in params:
+ params.pop("FLAML_sample_size")
+ return params
+
+
+class TransformersEstimator(BaseEstimator):
+ """The class for fine-tuning language models, using huggingface transformers API."""
+
+ ITER_HP = "global_max_steps"
+
+ def __init__(self, task="seq-classification", **config):
+ super().__init__(task, **config)
+ import uuid
+
+ self.trial_id = str(uuid.uuid1().hex)[:8]
+ if task not in NLG_TASKS: # TODO: not in NLG_TASKS
+ from .nlp.huggingface.training_args import (
+ TrainingArgumentsForAuto as TrainingArguments,
+ )
+ else:
+ from .nlp.huggingface.training_args import (
+ Seq2SeqTrainingArgumentsForAuto as TrainingArguments,
+ )
+ self._TrainingArguments = TrainingArguments
+
+ @classmethod
+ def search_space(cls, data_size, task, **params):
+ search_space_dict = {
+ "learning_rate": {
+ "domain": tune.loguniform(1e-6, 1e-4),
+ "init_value": 1e-5,
+ },
+ "num_train_epochs": {
+ "domain": tune.choice([1, 2, 3, 4, 5]),
+ "init_value": 3, # to be consistent with roberta
+ "low_cost_init_value": 1,
+ },
+ "per_device_train_batch_size": {
+ "domain": tune.choice([4, 8, 16, 32, 64]),
+ "init_value": 32,
+ "low_cost_init_value": 64,
+ },
+ "seed": {
+ "domain": tune.choice(range(1, 40)),
+ "init_value": 20,
+ },
+ "global_max_steps": {
+ "domain": sys.maxsize,
+ "init_value": sys.maxsize,
+ },
+ }
+
+ return search_space_dict
+
+ @property
+ def fp16(self):
+ return self._kwargs.get("gpu_per_trial") and self._training_args.fp16
+
+ @property
+ def no_cuda(self):
+ return not self._kwargs.get("gpu_per_trial")
+
+ def _set_training_args(self, **kwargs):
+ from .nlp.utils import date_str, Counter
+
+ for (key, val) in kwargs.items():
+ assert key not in self.params, (
+ "Since {} is in the search space, it cannot exist in 'custom_fit_kwargs' at the same time."
+ "If you need to fix the value of {} to {}, the only way is to add a single-value domain in the search "
+ "space by adding:\n '{}': {{ 'domain': {} }} to 'custom_hp'. For example:"
+ 'automl_settings["custom_hp"] = {{ "transformer": {{ "model_path": {{ "domain" : '
+ '"google/electra-small-discriminator" }} }} }}'.format(
+ key, key, val, key, val
+ )
+ )
+
+ """
+ If use has specified any custom args for TrainingArguments, update these arguments
+ """
+ self._training_args = self._TrainingArguments(**kwargs)
+
+ """
+ Update the attributes in TrainingArguments with self.params values
+ """
+ for key, val in self.params.items():
+ if hasattr(self._training_args, key):
+ setattr(self._training_args, key, val)
+
+ """
+ Update the attributes in TrainingArguments that depends on the values of self.params
+ """
+ local_dir = os.path.join(
+ self._training_args.output_dir, "train_{}".format(date_str())
+ )
+ if self._use_ray is True:
+ import ray
+
+ self._training_args.output_dir = ray.tune.get_trial_dir()
+ else:
+ self._training_args.output_dir = Counter.get_trial_fold_name(
+ local_dir, self.params, self.trial_id
+ )
+
+ self._training_args.fp16 = self.fp16
+ self._training_args.no_cuda = self.no_cuda
+
+ if (
+ self._task == TOKENCLASSIFICATION
+ and self._training_args.max_seq_length is not None
+ ):
+ logger.warning(
+ "For token classification task, FLAML currently does not support customizing the max_seq_length, max_seq_length will be reset to None."
+ )
+ setattr(self._training_args, "max_seq_length", None)
+
+ def _tokenize_text(self, X, y=None, **kwargs):
+ from .nlp.huggingface.utils import tokenize_text
+ from .nlp.utils import is_a_list_of_str
+
+ is_str = str(X.dtypes[0]) in ("string", "str")
+ is_list_of_str = is_a_list_of_str(X[list(X.keys())[0]].to_list()[0])
+
+ if is_str or is_list_of_str:
+ return tokenize_text(
+ X=X,
+ Y=y,
+ task=self._task,
+ hf_args=self._training_args,
+ tokenizer=self.tokenizer,
+ )
+ else:
+ return X, y
+
+ def _model_init(self):
+ from .nlp.huggingface.utils import load_model
+
+ this_model = load_model(
+ checkpoint_path=self._training_args.model_path,
+ task=self._task,
+ num_labels=self.num_labels,
+ )
+ return this_model
+
+ def _preprocess_data(self, X, y):
+ from datasets import Dataset
+
+ processed_X, processed_y_df = self._tokenize_text(X=X, y=y, **self._kwargs)
+ # convert y from pd.DataFrame back to pd.Series
+ processed_y = processed_y_df.iloc[:, 0]
+
+ processed_dataset = Dataset.from_pandas(processed_X.join(processed_y_df))
+
+ return processed_dataset, processed_X, processed_y
+
+ @property
+ def num_labels(self):
+ if self._task == SEQREGRESSION:
+ return 1
+ elif self._task == SEQCLASSIFICATION:
+ return len(set(self._y_train))
+ elif self._task == TOKENCLASSIFICATION:
+ return len(self._training_args.label_list)
+ else:
+ return None
+
+ @property
+ def tokenizer(self):
+ from transformers import AutoTokenizer
+
+ if self._task == SUMMARIZATION:
+ return AutoTokenizer.from_pretrained(
+ pretrained_model_name_or_path=self._training_args.model_path,
+ cache_dir=None,
+ use_fast=True,
+ revision="main",
+ use_auth_token=None,
+ )
+ else:
+ return AutoTokenizer.from_pretrained(
+ self._training_args.model_path,
+ use_fast=True,
+ add_prefix_space=self._add_prefix_space,
+ )
+
+ @property
+ def data_collator(self):
+ from .nlp.huggingface.data_collator import task_to_datacollator_class
+
+ data_collator_class = task_to_datacollator_class.get(self._task)
+
+ if data_collator_class:
+ kwargs = {
+ "model": self._model_init(),
+ # need to set model, or there's ValueError: Expected input batch_size (..) to match target batch_size (..)
+ "label_pad_token_id": -100, # pad with token id -100
+ "pad_to_multiple_of": 8,
+ # pad to multiple of 8 because quote Transformers: "This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta)"
+ "tokenizer": self.tokenizer,
+ }
+
+ for key in list(kwargs.keys()):
+ if (
+ key not in data_collator_class.__dict__.keys()
+ and key != "tokenizer"
+ ):
+ del kwargs[key]
+ return data_collator_class(**kwargs)
+ else:
+ return None
+
+ def fit(
+ self,
+ X_train: DataFrame,
+ y_train: Series,
+ budget=None,
+ free_mem_ratio=0,
+ X_val=None,
+ y_val=None,
+ gpu_per_trial=None,
+ metric=None,
+ **kwargs,
+ ):
+ import transformers
+
+ transformers.logging.set_verbosity_error()
+
+ from transformers import TrainerCallback
+ from transformers.trainer_utils import set_seed
+ from .nlp.huggingface.trainer import TrainerForAuto
+
+ try:
+ from ray.tune import is_session_enabled
+
+ self._use_ray = is_session_enabled()
+ except ImportError:
+ self._use_ray = False
+
+ this_params = self.params
+ self._kwargs = kwargs
+
+ self._X_train, self._y_train = X_train, y_train
+ self._set_training_args(**kwargs)
+ self._add_prefix_space = (
+ "roberta" in self._training_args.model_path
+ ) # If using roberta model, must set add_prefix_space to True to avoid the assertion error at
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249
+
+ train_dataset, self._X_train, self._y_train = self._preprocess_data(
+ X_train, y_train
+ )
+ if X_val is not None:
+ eval_dataset, self._X_val, self._y_val = self._preprocess_data(X_val, y_val)
+ else:
+ eval_dataset, self._X_val, self._y_val = None, None, None
+
+ set_seed(self.params.get("seed", self._training_args.seed))
+ self._metric = metric
+
+ class EarlyStoppingCallbackForAuto(TrainerCallback):
+ def on_train_begin(self, args, state, control, **callback_kwargs):
+ self.train_begin_time = time.time()
+
+ def on_step_begin(self, args, state, control, **callback_kwargs):
+ self.step_begin_time = time.time()
+
+ def on_step_end(self, args, state, control, **callback_kwargs):
+ if state.global_step == 1:
+ self.time_per_iter = time.time() - self.step_begin_time
+ if (
+ budget
+ and (
+ time.time() + self.time_per_iter
+ > self.train_begin_time + budget
+ )
+ or state.global_step >= this_params[TransformersEstimator.ITER_HP]
+ ):
+ control.should_training_stop = True
+ control.should_save = True
+ control.should_evaluate = True
+ return control
+
+ def on_epoch_end(self, args, state, control, **callback_kwargs):
+ if (
+ control.should_training_stop
+ or state.epoch + 1 >= args.num_train_epochs
+ ):
+ control.should_save = True
+ control.should_evaluate = True
+
+ self._trainer = TrainerForAuto(
+ args=self._training_args,
+ model_init=self._model_init,
+ train_dataset=train_dataset,
+ eval_dataset=eval_dataset,
+ tokenizer=self.tokenizer,
+ data_collator=self.data_collator,
+ compute_metrics=self._compute_metrics_by_dataset_name,
+ callbacks=[EarlyStoppingCallbackForAuto],
+ )
+
+ if self._task in NLG_TASKS:
+ setattr(self._trainer, "_is_seq2seq", True)
+
+ """
+ When not using ray for tuning, set the limit of CUDA_VISIBLE_DEVICES to math.ceil(gpu_per_trial),
+ so each estimator does not see all the GPUs
+ """
+ if gpu_per_trial is not None:
+ tmp_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "")
+ self._trainer.args._n_gpu = gpu_per_trial
+
+ # if gpu_per_trial == 0:
+ # os.environ["CUDA_VISIBLE_DEVICES"] = ""
+ if tmp_cuda_visible_devices.count(",") != math.ceil(gpu_per_trial) - 1:
+ os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(
+ [str(x) for x in range(math.ceil(gpu_per_trial))]
+ )
+
+ import time
+
+ start_time = time.time()
+ self._trainer.train()
+
+ if gpu_per_trial is not None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = tmp_cuda_visible_devices
+
+ self.params[self.ITER_HP] = self._trainer.state.global_step
+
+ self._checkpoint_path = self._select_checkpoint(self._trainer)
+ self._ckpt_remains = list(self._trainer.ckpt_to_metric.keys())
+
+ if hasattr(self._trainer, "intermediate_results"):
+ self.intermediate_results = [
+ x[1]
+ for x in sorted(
+ self._trainer.intermediate_results.items(), key=lambda x: x[0]
+ )
+ ]
+ self._trainer = None
+
+ return time.time() - start_time
+
+ def _delete_one_ckpt(self, ckpt_location):
+ if self._use_ray is False:
+ try:
+ shutil.rmtree(ckpt_location)
+ except FileNotFoundError:
+ logger.warning("checkpoint {} not found".format(ckpt_location))
+
+ def cleanup(self):
+ super().cleanup()
+ if hasattr(self, "_ckpt_remains"):
+ for each_ckpt in self._ckpt_remains:
+ self._delete_one_ckpt(each_ckpt)
+
+ def _select_checkpoint(self, trainer):
+ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+
+ if trainer.ckpt_to_metric:
+ best_ckpt, _ = min(
+ trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_automl_metric"]
+ )
+ best_ckpt_global_step = trainer.ckpt_to_global_step[best_ckpt]
+ for each_ckpt in list(trainer.ckpt_to_metric):
+ if each_ckpt != best_ckpt:
+ del trainer.ckpt_to_metric[each_ckpt]
+ del trainer.ckpt_to_global_step[each_ckpt]
+ self._delete_one_ckpt(each_ckpt)
+ else:
+ best_ckpt_global_step = trainer.state.global_step
+ best_ckpt = os.path.join(
+ trainer.args.output_dir,
+ f"{PREFIX_CHECKPOINT_DIR}-{best_ckpt_global_step}",
+ )
+ self.params[self.ITER_HP] = best_ckpt_global_step
+ logger.debug(trainer.state.global_step)
+ logger.debug(trainer.ckpt_to_global_step)
+ return best_ckpt
+
+ def _compute_metrics_by_dataset_name(self, eval_pred):
+ # TODO: call self._metric(eval_pred, self)
+ if isinstance(self._metric, str):
+ from .ml import metric_loss_score
+ from .nlp.huggingface.utils import postprocess_prediction_and_true
+
+ predictions, y_true = eval_pred
+ # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text
+ processed_predictions, processed_y_true = postprocess_prediction_and_true(
+ task=self._task,
+ y_pred=predictions,
+ tokenizer=self.tokenizer,
+ hf_args=self._training_args,
+ y_true=y_true,
+ )
+ metric_dict = {
+ "automl_metric": metric_loss_score(
+ metric_name=self._metric,
+ y_processed_predict=processed_predictions,
+ y_processed_true=processed_y_true,
+ labels=self._training_args.label_list,
+ )
+ }
+ else:
+ # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)
+ loss, metric_dict = self._metric(
+ X_test=self._X_val,
+ y_test=self._y_val,
+ estimator=self,
+ labels=None,
+ X_train=self._X_train,
+ y_train=self._y_train,
+ )
+ metric_dict["automl_metric"] = loss
+
+ return metric_dict
+
+ def _init_model_for_predict(self):
+ from .nlp.huggingface.trainer import TrainerForAuto
+
+ """
+ Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistent
+ with HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947
+ """
+ training_args = self._TrainingArguments(
+ local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16
+ )
+ for key, val in self._training_args.__dict__.items():
+ if key not in ("local_rank", "model_path", "fp16"):
+ setattr(training_args, key, val)
+ self._training_args = training_args
+
+ new_trainer = TrainerForAuto(
+ model=self._model_init(),
+ args=self._training_args,
+ data_collator=self.data_collator,
+ compute_metrics=self._compute_metrics_by_dataset_name,
+ )
+ if self._task in NLG_TASKS:
+ setattr(new_trainer, "_is_seq2seq", True)
+ return new_trainer
+
+ def predict_proba(self, X, **pred_kwargs):
+ from datasets import Dataset
+
+ if pred_kwargs:
+ for key, val in pred_kwargs.items():
+ setattr(self._training_args, key, val)
+
+ assert (
+ self._task in CLASSIFICATION
+ ), "predict_proba() only for classification tasks."
+
+ X_test, _ = self._tokenize_text(X, **self._kwargs)
+ test_dataset = Dataset.from_pandas(X_test)
+
+ new_trainer = self._init_model_for_predict()
+ predictions = new_trainer.predict(test_dataset)
+ return predictions.predictions
+
+ def score(self, X_val: DataFrame, y_val: Series, **kwargs):
+ import transformers
+
+ transformers.logging.set_verbosity_error()
+
+ self._metric = kwargs["metric"]
+
+ eval_dataset, X_val, y_val = self._preprocess_data(X_val, y_val)
+
+ new_trainer = self._init_model_for_predict()
+ return new_trainer.evaluate(eval_dataset)
+
+ def predict(self, X, **pred_kwargs):
+ import transformers
+ from datasets import Dataset
+ from .nlp.huggingface.utils import postprocess_prediction_and_true
+
+ transformers.logging.set_verbosity_error()
+
+ if pred_kwargs:
+ for key, val in pred_kwargs.items():
+ setattr(self._training_args, key, val)
+
+ X_test, _ = self._tokenize_text(X, **self._kwargs)
+ test_dataset = Dataset.from_pandas(X_test)
+
+ new_trainer = self._init_model_for_predict()
+
+ if self._task not in NLG_TASKS:
+ predictions = new_trainer.predict(test_dataset)
+ else:
+ predictions = new_trainer.predict(
+ test_dataset,
+ metric_key_prefix="predict",
+ )
+ post_y_pred, _ = postprocess_prediction_and_true(
+ task=self._task,
+ y_pred=predictions.predictions,
+ tokenizer=self.tokenizer,
+ hf_args=self._training_args,
+ X=X,
+ )
+ return post_y_pred
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ params[TransformersEstimator.ITER_HP] = params.get(
+ TransformersEstimator.ITER_HP, sys.maxsize
+ )
+ return params
+
+
+class TransformersEstimatorModelSelection(TransformersEstimator):
+ def __init__(self, task="seq-classification", **config):
+ super().__init__(task, **config)
+
+ @classmethod
+ def search_space(cls, data_size, task, **params):
+ search_space_dict = TransformersEstimator.search_space(
+ data_size, task, **params
+ )
+
+ """
+ For model selection, use the same search space regardless of memory constraint
+ If OOM, user should change the search space themselves
+ """
+
+ search_space_dict["model_path"] = {
+ "domain": tune.choice(
+ [
+ "google/electra-base-discriminator",
+ "bert-base-uncased",
+ "roberta-base",
+ "facebook/muppet-roberta-base",
+ "google/electra-small-discriminator",
+ ]
+ ),
+ "init_value": "facebook/muppet-roberta-base",
+ }
+ return search_space_dict
+
+
+class SKLearnEstimator(BaseEstimator):
+ """
+ The base class for tuning scikit-learn estimators.
+
+ Subclasses can modify the function signature of ``__init__`` to
+ ignore the values in ``config`` that are not relevant to the constructor
+ of their underlying estimator. For example, some regressors in ``scikit-learn``
+ don't accept the ``n_jobs`` parameter contained in ``config``. For these,
+ one can add ``n_jobs=None,`` before ``**config`` to make sure ``config`` doesn't
+ contain an ``n_jobs`` key.
+ """
+
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+
+ def _preprocess(self, X):
+ if isinstance(X, DataFrame):
+ cat_columns = X.select_dtypes(include=["category"]).columns
+ if not cat_columns.empty:
+ X = X.copy()
+ X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)
+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
+ # numpy array is not of numeric dtype
+ X = DataFrame(X)
+ for col in X.columns:
+ if isinstance(X[col][0], str):
+ X[col] = X[col].astype("category").cat.codes
+ X = X.to_numpy()
+ return X
+
+
+class LGBMEstimator(BaseEstimator):
+ """The class for tuning LGBM, using sklearn API."""
+
+ ITER_HP = "n_estimators"
+ HAS_CALLBACK = True
+ DEFAULT_ITER = 100
+
+ @classmethod
+ def search_space(cls, data_size, **params):
+ upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower
+ return {
+ "n_estimators": {
+ "domain": tune.lograndint(lower=4, upper=upper),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ "num_leaves": {
+ "domain": tune.lograndint(lower=4, upper=upper),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ "min_child_samples": {
+ "domain": tune.lograndint(lower=2, upper=2**7 + 1),
+ "init_value": 20,
+ },
+ "learning_rate": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1.0),
+ "init_value": 0.1,
+ },
+ "log_max_bin": { # log transformed with base 2
+ "domain": tune.lograndint(lower=3, upper=11),
+ "init_value": 8,
+ },
+ "colsample_bytree": {
+ "domain": tune.uniform(lower=0.01, upper=1.0),
+ "init_value": 1.0,
+ },
+ "reg_alpha": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),
+ "init_value": 1 / 1024,
+ },
+ "reg_lambda": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),
+ "init_value": 1.0,
+ },
+ }
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ if "log_max_bin" in params:
+ params["max_bin"] = (1 << params.pop("log_max_bin")) - 1
+ return params
+
+ @classmethod
+ def size(cls, config):
+ num_leaves = int(
+ round(
+ config.get("num_leaves")
+ or config.get("max_leaves")
+ or 1 << config.get("max_depth", 16)
+ )
+ )
+ n_estimators = int(round(config["n_estimators"]))
+ return (num_leaves * 3 + (num_leaves - 1) * 4 + 1.0) * n_estimators * 8
+
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+ if "verbose" not in self.params:
+ self.params["verbose"] = -1
+ if "regression" == task:
+ from lightgbm import LGBMRegressor
+
+ self.estimator_class = LGBMRegressor
+ elif "rank" == task:
+ from lightgbm import LGBMRanker
+
+ self.estimator_class = LGBMRanker
+ else:
+ from lightgbm import LGBMClassifier
+
+ self.estimator_class = LGBMClassifier
+ self._time_per_iter = None
+ self._train_size = 0
+ self._mem_per_iter = -1
+ self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0, 0) is not None
+
+ def _preprocess(self, X):
+ if (
+ not isinstance(X, DataFrame)
+ and issparse(X)
+ and np.issubdtype(X.dtype, np.integer)
+ ):
+ X = X.astype(float)
+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
+ # numpy array is not of numeric dtype
+ X = DataFrame(X)
+ for col in X.columns:
+ if isinstance(X[col][0], str):
+ X[col] = X[col].astype("category").cat.codes
+ X = X.to_numpy()
+ return X
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ start_time = time.time()
+ deadline = start_time + budget if budget else np.inf
+ n_iter = self.params.get(self.ITER_HP, self.DEFAULT_ITER)
+ trained = False
+ if not self.HAS_CALLBACK:
+ mem0 = psutil.virtual_memory().available if psutil is not None else 1
+ if (
+ (
+ not self._time_per_iter
+ or abs(self._train_size - X_train.shape[0]) > 4
+ )
+ and budget is not None
+ or self._mem_per_iter < 0
+ and psutil is not None
+ ) and n_iter > 1:
+ self.params[self.ITER_HP] = 1
+ self._t1 = self._fit(X_train, y_train, **kwargs)
+ if budget is not None and self._t1 >= budget or n_iter == 1:
+ return self._t1
+ mem1 = psutil.virtual_memory().available if psutil is not None else 1
+ self._mem1 = mem0 - mem1
+ self.params[self.ITER_HP] = min(n_iter, 4)
+ self._t2 = self._fit(X_train, y_train, **kwargs)
+ mem2 = psutil.virtual_memory().available if psutil is not None else 1
+ self._mem2 = max(mem0 - mem2, self._mem1)
+ # if self._mem1 <= 0:
+ # self._mem_per_iter = self._mem2 / (self.params[self.ITER_HP] + 1)
+ # elif self._mem2 <= 0:
+ # self._mem_per_iter = self._mem1
+ # else:
+ self._mem_per_iter = min(
+ self._mem1, self._mem2 / self.params[self.ITER_HP]
+ )
+ # if self._mem_per_iter <= 1 and psutil is not None:
+ # n_iter = self.params[self.ITER_HP]
+ self._time_per_iter = (
+ (self._t2 - self._t1) / (self.params[self.ITER_HP] - 1)
+ if self._t2 > self._t1
+ else self._t1
+ if self._t1
+ else 0.001
+ )
+ self._train_size = X_train.shape[0]
+ if (
+ budget is not None
+ and self._t1 + self._t2 >= budget
+ or n_iter == self.params[self.ITER_HP]
+ ):
+ # self.params[self.ITER_HP] = n_iter
+ return time.time() - start_time
+ trained = True
+ # logger.debug(mem0)
+ # logger.debug(self._mem_per_iter)
+ if n_iter > 1:
+ max_iter = min(
+ n_iter,
+ int(
+ (budget - time.time() + start_time - self._t1)
+ / self._time_per_iter
+ + 1
+ )
+ if budget is not None
+ else n_iter,
+ int((1 - free_mem_ratio) * mem0 / self._mem_per_iter)
+ if psutil is not None and self._mem_per_iter > 0
+ else n_iter,
+ )
+ if trained and max_iter <= self.params[self.ITER_HP]:
+ return time.time() - start_time
+ # when not trained, train at least one iter
+ self.params[self.ITER_HP] = max(max_iter, 1)
+ if self.HAS_CALLBACK:
+ kwargs_callbacks = kwargs.get("callbacks")
+ if kwargs_callbacks:
+ callbacks = kwargs_callbacks + self._callbacks(
+ start_time, deadline, free_mem_ratio
+ )
+ kwargs.pop("callbacks")
+ else:
+ callbacks = self._callbacks(start_time, deadline, free_mem_ratio)
+ if isinstance(self, XGBoostSklearnEstimator):
+ from xgboost import __version__
+
+ if __version__ >= "1.6.0":
+ # since xgboost>=1.6.0, callbacks can't be passed in fit()
+ self.params["callbacks"] = callbacks
+ callbacks = None
+ self._fit(
+ X_train,
+ y_train,
+ callbacks=callbacks,
+ **kwargs,
+ )
+ if callbacks is None:
+ # for xgboost>=1.6.0, pop callbacks to enable pickle
+ callbacks = self.params.pop("callbacks")
+ self._model.set_params(callbacks=callbacks[:-1])
+ best_iteration = (
+ self._model.get_booster().best_iteration
+ if isinstance(self, XGBoostSklearnEstimator)
+ else self._model.best_iteration_
+ )
+ if best_iteration is not None:
+ self._model.set_params(n_estimators=best_iteration + 1)
+ else:
+ self._fit(X_train, y_train, **kwargs)
+ train_time = time.time() - start_time
+ return train_time
+
+ def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:
+ return [partial(self._callback, start_time, deadline, free_mem_ratio)]
+
+ def _callback(self, start_time, deadline, free_mem_ratio, env) -> None:
+ from lightgbm.callback import EarlyStopException
+
+ now = time.time()
+ if env.iteration == 0:
+ self._time_per_iter = now - start_time
+ if now + self._time_per_iter > deadline:
+ raise EarlyStopException(env.iteration, env.evaluation_result_list)
+ if psutil is not None:
+ mem = psutil.virtual_memory()
+ if mem.available / mem.total < free_mem_ratio:
+ raise EarlyStopException(env.iteration, env.evaluation_result_list)
+
+
+class XGBoostEstimator(SKLearnEstimator):
+ """The class for tuning XGBoost regressor, not using sklearn API."""
+
+ DEFAULT_ITER = 10
+
+ @classmethod
+ def search_space(cls, data_size, **params):
+ upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower
+ return {
+ "n_estimators": {
+ "domain": tune.lograndint(lower=4, upper=upper),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ "max_leaves": {
+ "domain": tune.lograndint(lower=4, upper=upper),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ "max_depth": {
+ "domain": tune.choice([0, 6, 12]),
+ "init_value": 0,
+ },
+ "min_child_weight": {
+ "domain": tune.loguniform(lower=0.001, upper=128),
+ "init_value": 1.0,
+ },
+ "learning_rate": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1.0),
+ "init_value": 0.1,
+ },
+ "subsample": {
+ "domain": tune.uniform(lower=0.1, upper=1.0),
+ "init_value": 1.0,
+ },
+ "colsample_bylevel": {
+ "domain": tune.uniform(lower=0.01, upper=1.0),
+ "init_value": 1.0,
+ },
+ "colsample_bytree": {
+ "domain": tune.uniform(lower=0.01, upper=1.0),
+ "init_value": 1.0,
+ },
+ "reg_alpha": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),
+ "init_value": 1 / 1024,
+ },
+ "reg_lambda": {
+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),
+ "init_value": 1.0,
+ },
+ }
+
+ @classmethod
+ def size(cls, config):
+ return LGBMEstimator.size(config)
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 1.6
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ max_depth = params["max_depth"] = params.get("max_depth", 0)
+ if max_depth == 0:
+ params["grow_policy"] = params.get("grow_policy", "lossguide")
+ params["tree_method"] = params.get("tree_method", "hist")
+ # params["booster"] = params.get("booster", "gbtree")
+ params["use_label_encoder"] = params.get("use_label_encoder", False)
+ if "n_jobs" in config:
+ params["nthread"] = params.pop("n_jobs")
+ return params
+
+ def __init__(
+ self,
+ task="regression",
+ **config,
+ ):
+ super().__init__(task, **config)
+ self.params["verbosity"] = 0
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ import xgboost as xgb
+
+ start_time = time.time()
+ deadline = start_time + budget if budget else np.inf
+ if issparse(X_train):
+ if xgb.__version__ < "1.6.0":
+ # "auto" fails for sparse input since xgboost 1.6.0
+ self.params["tree_method"] = "auto"
+ else:
+ X_train = self._preprocess(X_train)
+ if "sample_weight" in kwargs:
+ dtrain = xgb.DMatrix(X_train, label=y_train, weight=kwargs["sample_weight"])
+ else:
+ dtrain = xgb.DMatrix(X_train, label=y_train)
+
+ objective = self.params.get("objective")
+ if isinstance(objective, str):
+ obj = None
+ else:
+ obj = objective
+ if "objective" in self.params:
+ del self.params["objective"]
+ _n_estimators = self.params.pop("n_estimators")
+ callbacks = XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)
+ if callbacks:
+ self._model = xgb.train(
+ self.params,
+ dtrain,
+ _n_estimators,
+ obj=obj,
+ callbacks=callbacks,
+ )
+ self.params["n_estimators"] = self._model.best_iteration + 1
+ else:
+ self._model = xgb.train(self.params, dtrain, _n_estimators, obj=obj)
+ self.params["n_estimators"] = _n_estimators
+ self.params["objective"] = objective
+ del dtrain
+ train_time = time.time() - start_time
+ return train_time
+
+ def predict(self, X, **kwargs):
+ import xgboost as xgb
+
+ if not issparse(X):
+ X = self._preprocess(X)
+ dtest = xgb.DMatrix(X)
+ return super().predict(dtest, **kwargs)
+
+ @classmethod
+ def _callbacks(cls, start_time, deadline, free_mem_ratio):
+ try:
+ from xgboost.callback import TrainingCallback
+ except ImportError: # for xgboost<1.3
+ return None
+
+ class ResourceLimit(TrainingCallback):
+ def after_iteration(self, model, epoch, evals_log) -> bool:
+ now = time.time()
+ if epoch == 0:
+ self._time_per_iter = now - start_time
+ if now + self._time_per_iter > deadline:
+ return True
+ if psutil is not None:
+ mem = psutil.virtual_memory()
+ if mem.available / mem.total < free_mem_ratio:
+ return True
+ return False
+
+ return [ResourceLimit()]
+
+
+class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):
+ """The class for tuning XGBoost with unlimited depth, using sklearn API."""
+
+ DEFAULT_ITER = 10
+
+ @classmethod
+ def search_space(cls, data_size, **params):
+ space = XGBoostEstimator.search_space(data_size)
+ space.pop("max_depth")
+ return space
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return XGBoostEstimator.cost_relative2lgbm()
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ max_depth = params["max_depth"] = params.get("max_depth", 0)
+ if max_depth == 0:
+ params["grow_policy"] = params.get("grow_policy", "lossguide")
+ params["tree_method"] = params.get("tree_method", "hist")
+ params["use_label_encoder"] = params.get("use_label_encoder", False)
+ return params
+
+ def __init__(
+ self,
+ task="binary",
+ **config,
+ ):
+ super().__init__(task, **config)
+ del self.params["verbose"]
+ self.params["verbosity"] = 0
+ import xgboost as xgb
+
+ self.estimator_class = xgb.XGBRegressor
+ if "rank" == task:
+ self.estimator_class = xgb.XGBRanker
+ elif task in CLASSIFICATION:
+ self.estimator_class = xgb.XGBClassifier
+ self._xgb_version = xgb.__version__
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ if issparse(X_train) and self._xgb_version < "1.6.0":
+ # "auto" fails for sparse input since xgboost 1.6.0
+ self.params["tree_method"] = "auto"
+ if kwargs.get("gpu_per_trial"):
+ self.params["tree_method"] = "gpu_hist"
+ kwargs.pop("gpu_per_trial")
+ return super().fit(X_train, y_train, budget, free_mem_ratio, **kwargs)
+
+ def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:
+ return XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)
+
+
+class XGBoostLimitDepthEstimator(XGBoostSklearnEstimator):
+ """The class for tuning XGBoost with limited depth, using sklearn API."""
+
+ @classmethod
+ def search_space(cls, data_size, **params):
+ space = XGBoostEstimator.search_space(data_size)
+ space.pop("max_leaves")
+ upper = max(6, int(np.log2(data_size[0])))
+ space["max_depth"] = {
+ "domain": tune.randint(lower=1, upper=min(upper, 16)),
+ "init_value": 6,
+ "low_cost_init_value": 1,
+ }
+ space["learning_rate"]["init_value"] = 0.3
+ space["n_estimators"]["init_value"] = 10
+ return space
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 64
+
+
+class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
+ """The class for tuning Random Forest."""
+
+ HAS_CALLBACK = False
+ nrows = 101
+
+ @classmethod
+ def search_space(cls, data_size, task, **params):
+ RandomForestEstimator.nrows = int(data_size[0])
+ upper = min(2048, RandomForestEstimator.nrows)
+ init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1
+ lower = min(0.1, init)
+ space = {
+ "n_estimators": {
+ "domain": tune.lograndint(lower=4, upper=max(5, upper)),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ "max_features": {
+ "domain": tune.loguniform(lower=lower, upper=1.0),
+ "init_value": init,
+ },
+ "max_leaves": {
+ "domain": tune.lograndint(
+ lower=4,
+ upper=max(5, min(32768, RandomForestEstimator.nrows >> 1)), #
+ ),
+ "init_value": 4,
+ "low_cost_init_value": 4,
+ },
+ }
+ if task in CLASSIFICATION:
+ space["criterion"] = {
+ "domain": tune.choice(["gini", "entropy"]),
+ # "init_value": "gini",
+ }
+ return space
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 2
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ if "max_leaves" in params:
+ params["max_leaf_nodes"] = params.get(
+ "max_leaf_nodes", params.pop("max_leaves")
+ )
+ if self._task not in CLASSIFICATION and "criterion" in config:
+ params.pop("criterion")
+ if "random_state" not in params:
+ params["random_state"] = 12032022
+ return params
+
+ def __init__(
+ self,
+ task="binary",
+ **params,
+ ):
+ super().__init__(task, **params)
+ self.params["verbose"] = 0
+ self.estimator_class = RandomForestRegressor
+ if task in CLASSIFICATION:
+ self.estimator_class = RandomForestClassifier
+
+
+class ExtraTreesEstimator(RandomForestEstimator):
+ """The class for tuning Extra Trees."""
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 1.9
+
+ def __init__(self, task="binary", **params):
+ super().__init__(task, **params)
+ if "regression" in task:
+ self.estimator_class = ExtraTreesRegressor
+ else:
+ self.estimator_class = ExtraTreesClassifier
+
+
+class LRL1Classifier(SKLearnEstimator):
+ """The class for tuning Logistic Regression with L1 regularization."""
+
+ @classmethod
+ def search_space(cls, **params):
+ return {
+ "C": {
+ "domain": tune.loguniform(lower=0.03125, upper=32768.0),
+ "init_value": 1.0,
+ },
+ }
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 160
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ params["tol"] = params.get("tol", 0.0001)
+ params["solver"] = params.get("solver", "saga")
+ params["penalty"] = params.get("penalty", "l1")
+ return params
+
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+ assert task in CLASSIFICATION, "LogisticRegression for classification task only"
+ self.estimator_class = LogisticRegression
+
+
+class LRL2Classifier(SKLearnEstimator):
+ """The class for tuning Logistic Regression with L2 regularization."""
+
+ limit_resource = True
+
+ @classmethod
+ def search_space(cls, **params):
+ return LRL1Classifier.search_space(**params)
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 25
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ params["tol"] = params.get("tol", 0.0001)
+ params["solver"] = params.get("solver", "lbfgs")
+ params["penalty"] = params.get("penalty", "l2")
+ return params
+
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+ assert task in CLASSIFICATION, "LogisticRegression for classification task only"
+ self.estimator_class = LogisticRegression
+
+
+class CatBoostEstimator(BaseEstimator):
+ """The class for tuning CatBoost."""
+
+ ITER_HP = "n_estimators"
+ DEFAULT_ITER = 1000
+
+ @classmethod
+ def search_space(cls, data_size, **params):
+ upper = max(min(round(1500000 / data_size[0]), 150), 12)
+ return {
+ "early_stopping_rounds": {
+ "domain": tune.lograndint(lower=10, upper=upper),
+ "init_value": 10,
+ "low_cost_init_value": 10,
+ },
+ "learning_rate": {
+ "domain": tune.loguniform(lower=0.005, upper=0.2),
+ "init_value": 0.1,
+ },
+ "n_estimators": {
+ "domain": 8192,
+ "init_value": 8192,
+ },
+ }
+
+ @classmethod
+ def size(cls, config):
+ n_estimators = config.get("n_estimators", 8192)
+ max_leaves = 64
+ return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 15
+
+ def _preprocess(self, X):
+ if isinstance(X, DataFrame):
+ cat_columns = X.select_dtypes(include=["category"]).columns
+ if not cat_columns.empty:
+ X = X.copy()
+ X[cat_columns] = X[cat_columns].apply(
+ lambda x: x.cat.rename_categories(
+ [
+ str(c) if isinstance(c, float) else c
+ for c in x.cat.categories
+ ]
+ )
+ )
+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
+ # numpy array is not of numeric dtype
+ X = DataFrame(X)
+ for col in X.columns:
+ if isinstance(X[col][0], str):
+ X[col] = X[col].astype("category").cat.codes
+ X = X.to_numpy()
+ return X
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ params["n_estimators"] = params.get("n_estimators", 8192)
+ if "n_jobs" in params:
+ params["thread_count"] = params.pop("n_jobs")
+ return params
+
+ def __init__(
+ self,
+ task="binary",
+ **config,
+ ):
+ super().__init__(task, **config)
+ self.params.update(
+ {
+ "verbose": config.get("verbose", False),
+ "random_seed": config.get("random_seed", 10242048),
+ }
+ )
+ from catboost import CatBoostRegressor
+
+ self.estimator_class = CatBoostRegressor
+ if task in CLASSIFICATION:
+ from catboost import CatBoostClassifier
+
+ self.estimator_class = CatBoostClassifier
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ start_time = time.time()
+ deadline = start_time + budget if budget else np.inf
+ train_dir = f"catboost_{str(start_time)}"
+ X_train = self._preprocess(X_train)
+ if isinstance(X_train, DataFrame):
+ cat_features = list(X_train.select_dtypes(include="category").columns)
+ else:
+ cat_features = []
+ use_best_model = kwargs.get("use_best_model", True)
+ n = (
+ max(int(len(y_train) * 0.9), len(y_train) - 1000)
+ if use_best_model
+ else len(y_train)
+ )
+ X_tr, y_tr = X_train[:n], y_train[:n]
+ from catboost import Pool, __version__
+
+ eval_set = (
+ Pool(data=X_train[n:], label=y_train[n:], cat_features=cat_features)
+ if use_best_model
+ else None
+ )
+ if "sample_weight" in kwargs:
+ weight = kwargs["sample_weight"]
+ if weight is not None:
+ kwargs["sample_weight"] = weight[:n]
+ else:
+ weight = None
+
+ model = self.estimator_class(train_dir=train_dir, **self.params)
+ if __version__ >= "0.26":
+ model.fit(
+ X_tr,
+ y_tr,
+ cat_features=cat_features,
+ eval_set=eval_set,
+ callbacks=CatBoostEstimator._callbacks(
+ start_time, deadline, free_mem_ratio if use_best_model else None
+ ),
+ **kwargs,
+ )
+ else:
+ model.fit(
+ X_tr,
+ y_tr,
+ cat_features=cat_features,
+ eval_set=eval_set,
+ **kwargs,
+ )
+ shutil.rmtree(train_dir, ignore_errors=True)
+ if weight is not None:
+ kwargs["sample_weight"] = weight
+ self._model = model
+ self.params[self.ITER_HP] = self._model.tree_count_
+ train_time = time.time() - start_time
+ return train_time
+
+ @classmethod
+ def _callbacks(cls, start_time, deadline, free_mem_ratio):
+ class ResourceLimit:
+ def after_iteration(self, info) -> bool:
+ now = time.time()
+ if info.iteration == 1:
+ self._time_per_iter = now - start_time
+ if now + self._time_per_iter > deadline:
+ return False
+ if psutil is not None and free_mem_ratio is not None:
+ mem = psutil.virtual_memory()
+ if mem.available / mem.total < free_mem_ratio:
+ return False
+ return True # can continue
+
+ return [ResourceLimit()]
+
+
+class KNeighborsEstimator(BaseEstimator):
+ @classmethod
+ def search_space(cls, data_size, **params):
+ upper = min(512, int(data_size[0] / 2))
+ return {
+ "n_neighbors": {
+ "domain": tune.lograndint(lower=1, upper=max(2, upper)),
+ "init_value": 5,
+ "low_cost_init_value": 1,
+ },
+ }
+
+ @classmethod
+ def cost_relative2lgbm(cls):
+ return 30
+
+ def config2params(self, config: dict) -> dict:
+ params = super().config2params(config)
+ params["weights"] = params.get("weights", "distance")
+ return params
+
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+ if task in CLASSIFICATION:
+ from sklearn.neighbors import KNeighborsClassifier
+
+ self.estimator_class = KNeighborsClassifier
+ else:
+ from sklearn.neighbors import KNeighborsRegressor
+
+ self.estimator_class = KNeighborsRegressor
+
+ def _preprocess(self, X):
+ if isinstance(X, DataFrame):
+ cat_columns = X.select_dtypes(["category"]).columns
+ if X.shape[1] == len(cat_columns):
+ raise ValueError("kneighbor requires at least one numeric feature")
+ X = X.drop(cat_columns, axis=1)
+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
+ # drop categocial columns if any
+ X = DataFrame(X)
+ cat_columns = []
+ for col in X.columns:
+ if isinstance(X[col][0], str):
+ cat_columns.append(col)
+ X = X.drop(cat_columns, axis=1)
+ X = X.to_numpy()
+ return X
+
+
+class Prophet(SKLearnEstimator):
+ """The class for tuning Prophet."""
+
+ @classmethod
+ def search_space(cls, **params):
+ space = {
+ "changepoint_prior_scale": {
+ "domain": tune.loguniform(lower=0.001, upper=0.05),
+ "init_value": 0.05,
+ "low_cost_init_value": 0.001,
+ },
+ "seasonality_prior_scale": {
+ "domain": tune.loguniform(lower=0.01, upper=10),
+ "init_value": 10,
+ },
+ "holidays_prior_scale": {
+ "domain": tune.loguniform(lower=0.01, upper=10),
+ "init_value": 10,
+ },
+ "seasonality_mode": {
+ "domain": tune.choice(["additive", "multiplicative"]),
+ "init_value": "multiplicative",
+ },
+ }
+ return space
+
+ def __init__(self, task="ts_forecast", n_jobs=1, **params):
+ super().__init__(task, **params)
+
+ def _join(self, X_train, y_train):
+ assert TS_TIMESTAMP_COL in X_train, (
+ "Dataframe for training ts_forecast model must have column"
+ f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'
+ )
+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
+ train_df = X_train.join(y_train)
+ return train_df
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ from prophet import Prophet
+
+ current_time = time.time()
+ train_df = self._join(X_train, y_train)
+ train_df = self._preprocess(train_df)
+ cols = list(train_df)
+ cols.remove(TS_TIMESTAMP_COL)
+ cols.remove(TS_VALUE_COL)
+ logging.getLogger("prophet").setLevel(logging.WARNING)
+ model = Prophet(**self.params)
+ for regressor in cols:
+ model.add_regressor(regressor)
+ with suppress_stdout_stderr():
+ model.fit(train_df)
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+ def predict(self, X, **kwargs):
+ if isinstance(X, int):
+ raise ValueError(
+ "predict() with steps is only supported for arima/sarimax."
+ " For Prophet, pass a dataframe with the first column containing"
+ " the timestamp values."
+ )
+ if self._model is not None:
+ X = self._preprocess(X)
+ forecast = self._model.predict(X, **kwargs)
+ return forecast["yhat"]
+ else:
+ logger.warning(
+ "Estimator is not fit yet. Please run fit() before predict()."
+ )
+ return np.ones(X.shape[0])
+
+ def score(self, X_val: DataFrame, y_val: Series, **kwargs):
+ from sklearn.metrics import r2_score
+ from .ml import metric_loss_score
+
+ y_pred = self.predict(X_val, **kwargs)
+ self._metric = kwargs.get("metric", None)
+ if self._metric:
+ return metric_loss_score(self._metric, y_pred, y_val)
+ else:
+ return r2_score(y_pred, y_val)
+
+
+class ARIMA(Prophet):
+ """The class for tuning ARIMA."""
+
+ @classmethod
+ def search_space(cls, **params):
+ space = {
+ "p": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 2,
+ "low_cost_init_value": 0,
+ },
+ "d": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 2,
+ "low_cost_init_value": 0,
+ },
+ "q": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ }
+ return space
+
+ def _join(self, X_train, y_train):
+ train_df = super()._join(X_train, y_train)
+ train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])
+ train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)
+ return train_df
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ import warnings
+
+ warnings.filterwarnings("ignore")
+ from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator
+
+ current_time = time.time()
+ train_df = self._join(X_train, y_train)
+ train_df = self._preprocess(train_df)
+ regressors = list(train_df)
+ regressors.remove(TS_VALUE_COL)
+ if regressors:
+ model = ARIMA_estimator(
+ train_df[[TS_VALUE_COL]],
+ exog=train_df[regressors],
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ else:
+ model = ARIMA_estimator(
+ train_df,
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ with suppress_stdout_stderr():
+ model = model.fit()
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+ def predict(self, X, **kwargs):
+ if self._model is not None:
+ if isinstance(X, int):
+ forecast = self._model.forecast(steps=X)
+ elif isinstance(X, DataFrame):
+ start = X[TS_TIMESTAMP_COL].iloc[0]
+ end = X[TS_TIMESTAMP_COL].iloc[-1]
+ if len(X.columns) > 1:
+ X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))
+ regressors = list(X)
+ forecast = self._model.predict(
+ start=start, end=end, exog=X[regressors], **kwargs
+ )
+ else:
+ forecast = self._model.predict(start=start, end=end, **kwargs)
+ else:
+ raise ValueError(
+ "X needs to be either a pandas Dataframe with dates as the first column"
+ " or an int number of periods for predict()."
+ )
+ return forecast
+ else:
+ return np.ones(X if isinstance(X, int) else X.shape[0])
+
+
+class SARIMAX(ARIMA):
+ """The class for tuning SARIMA."""
+
+ @classmethod
+ def search_space(cls, **params):
+ space = {
+ "p": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 2,
+ "low_cost_init_value": 0,
+ },
+ "d": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 2,
+ "low_cost_init_value": 0,
+ },
+ "q": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "P": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "D": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "Q": {
+ "domain": tune.qrandint(lower=0, upper=10, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "s": {
+ "domain": tune.choice([1, 4, 6, 12]),
+ "init_value": 12,
+ },
+ }
+ return space
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ import warnings
+
+ warnings.filterwarnings("ignore")
+ from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator
+
+ current_time = time.time()
+ train_df = self._join(X_train, y_train)
+ train_df = self._preprocess(train_df)
+ regressors = list(train_df)
+ regressors.remove(TS_VALUE_COL)
+ if regressors:
+ model = SARIMAX_estimator(
+ train_df[[TS_VALUE_COL]],
+ exog=train_df[regressors],
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ seasonal_order=(
+ self.params["P"],
+ self.params["D"],
+ self.params["Q"],
+ self.params["s"],
+ ),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ else:
+ model = SARIMAX_estimator(
+ train_df,
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ seasonal_order=(
+ self.params["P"],
+ self.params["D"],
+ self.params["Q"],
+ self.params["s"],
+ ),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ with suppress_stdout_stderr():
+ model = model.fit()
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+
+class TS_SKLearn(SKLearnEstimator):
+ """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""
+
+ base_class = SKLearnEstimator
+
+ @classmethod
+ def search_space(cls, data_size, pred_horizon, **params):
+ space = cls.base_class.search_space(data_size, **params)
+ space.update(
+ {
+ "optimize_for_horizon": {
+ "domain": tune.choice([True, False]),
+ "init_value": False,
+ "low_cost_init_value": False,
+ },
+ "lags": {
+ "domain": tune.randint(
+ lower=1, upper=max(2, int(np.sqrt(data_size[0])))
+ ),
+ "init_value": 3,
+ },
+ }
+ )
+ return space
+
+ def __init__(self, task="ts_forecast", **params):
+ super().__init__(task, **params)
+ self.hcrystaball_model = None
+ self.ts_task = (
+ "regression" if task in TS_FORECASTREGRESSION else "classification"
+ )
+
+ def transform_X(self, X):
+ cols = list(X)
+ if len(cols) == 1:
+ ds_col = cols[0]
+ X = DataFrame(index=X[ds_col])
+ elif len(cols) > 1:
+ ds_col = cols[0]
+ exog_cols = cols[1:]
+ X = X[exog_cols].set_index(X[ds_col])
+ return X
+
+ def _fit(self, X_train, y_train, budget=None, **kwargs):
+ from hcrystalball.wrappers import get_sklearn_wrapper
+
+ X_train = self.transform_X(X_train)
+ X_train = self._preprocess(X_train)
+ params = self.params.copy()
+ lags = params.pop("lags")
+ optimize_for_horizon = params.pop("optimize_for_horizon")
+ estimator = self.base_class(task=self.ts_task, **params)
+ self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)
+ self.hcrystaball_model.lags = int(lags)
+ self.hcrystaball_model.fit(X_train, y_train)
+ if optimize_for_horizon:
+ # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon
+ model_list = []
+ for i in range(1, kwargs["period"] + 1):
+ (
+ X_fit,
+ y_fit,
+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
+ X_train, y_train, i
+ )
+ self.hcrystaball_model.model.set_params(**estimator.params)
+ model = self.hcrystaball_model.model.fit(X_fit, y_fit)
+ model_list.append(model)
+ self._model = model_list
+ else:
+ (
+ X_fit,
+ y_fit,
+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
+ X_train, y_train, kwargs["period"]
+ )
+ self.hcrystaball_model.model.set_params(**estimator.params)
+ model = self.hcrystaball_model.model.fit(X_fit, y_fit)
+ self._model = model
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ current_time = time.time()
+ self._fit(X_train, y_train, budget=budget, **kwargs)
+ train_time = time.time() - current_time
+ return train_time
+
+ def predict(self, X, **kwargs):
+ if self._model is not None:
+ X = self.transform_X(X)
+ X = self._preprocess(X)
+ if isinstance(self._model, list):
+ assert len(self._model) == len(
+ X
+ ), "Model is optimized for horizon, length of X must be equal to `period`."
+ preds = []
+ for i in range(1, len(self._model) + 1):
+ (
+ X_pred,
+ _,
+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
+ X.iloc[:i, :]
+ )
+ preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])
+ forecast = DataFrame(
+ data=np.asarray(preds).reshape(-1, 1),
+ columns=[self.hcrystaball_model.name],
+ index=X.index,
+ )
+ else:
+ (
+ X_pred,
+ _,
+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)
+ forecast = self._model.predict(X_pred, **kwargs)
+ return forecast
+ else:
+ logger.warning(
+ "Estimator is not fit yet. Please run fit() before predict()."
+ )
+ return np.ones(X.shape[0])
+
+
+class LGBM_TS(TS_SKLearn):
+ """The class for tuning LGBM Regressor for time-series forecasting"""
+
+ base_class = LGBMEstimator
+
+
+class XGBoost_TS(TS_SKLearn):
+ """The class for tuning XGBoost Regressor for time-series forecasting"""
+
+ base_class = XGBoostSklearnEstimator
+
+
+# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball
+# class CatBoost_TS_Regressor(TS_Regressor):
+# base_class = CatBoostEstimator
+
+
+class RF_TS(TS_SKLearn):
+ """The class for tuning Random Forest Regressor for time-series forecasting"""
+
+ base_class = RandomForestEstimator
+
+
+class ExtraTrees_TS(TS_SKLearn):
+ """The class for tuning Extra Trees Regressor for time-series forecasting"""
+
+ base_class = ExtraTreesEstimator
+
+
+class XGBoostLimitDepth_TS(TS_SKLearn):
+ """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""
+
+ base_class = XGBoostLimitDepthEstimator
+
+
+class TemporalFusionTransformerEstimator(SKLearnEstimator):
+ """The class for tuning Temporal Fusion Transformer"""
+
+ @classmethod
+ def search_space(cls, data_size, pred_horizon, **params):
+ space = {
+ "gradient_clip_val": {
+ "domain": tune.loguniform(lower=0.01, upper=100.0),
+ "init_value": 0.01,
+ },
+ "hidden_size": {
+ "domain": tune.lograndint(lower=8, upper=512),
+ "init_value": 16,
+ },
+ "hidden_continuous_size": {
+ "domain": tune.randint(lower=1, upper=65),
+ "init_value": 8,
+ },
+ "attention_head_size": {
+ "domain": tune.randint(lower=1, upper=5),
+ "init_value": 4,
+ },
+ "dropout": {
+ "domain": tune.uniform(lower=0.1, upper=0.3),
+ "init_value": 0.1,
+ },
+ "learning_rate": {
+ "domain": tune.loguniform(lower=0.00001, upper=1.0),
+ "init_value": 0.001,
+ },
+ }
+ return space
+
+ def transform_ds(self, X_train, y_train, **kwargs):
+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
+ self.data = X_train.join(y_train)
+
+ max_prediction_length = kwargs["period"]
+ self.max_encoder_length = kwargs["max_encoder_length"]
+ training_cutoff = self.data["time_idx"].max() - max_prediction_length
+
+ from pytorch_forecasting import TimeSeriesDataSet
+ from pytorch_forecasting.data import GroupNormalizer
+
+ self.group_ids = kwargs["group_ids"].copy()
+ training = TimeSeriesDataSet(
+ self.data[lambda x: x.time_idx <= training_cutoff],
+ time_idx="time_idx",
+ target=TS_VALUE_COL,
+ group_ids=self.group_ids,
+ min_encoder_length=kwargs.get(
+ "min_encoder_length", self.max_encoder_length // 2
+ ), # keep encoder length long (as it is in the validation set)
+ max_encoder_length=self.max_encoder_length,
+ min_prediction_length=1,
+ max_prediction_length=max_prediction_length,
+ static_categoricals=kwargs.get("static_categoricals", []),
+ static_reals=kwargs.get("static_reals", []),
+ time_varying_known_categoricals=kwargs.get(
+ "time_varying_known_categoricals", []
+ ),
+ time_varying_known_reals=kwargs.get("time_varying_known_reals", []),
+ time_varying_unknown_categoricals=kwargs.get(
+ "time_varying_unknown_categoricals", []
+ ),
+ time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),
+ variable_groups=kwargs.get(
+ "variable_groups", {}
+ ), # group of categorical variables can be treated as one variable
+ lags=kwargs.get("lags", {}),
+ target_normalizer=GroupNormalizer(
+ groups=kwargs["group_ids"], transformation="softplus"
+ ), # use softplus and normalize by group
+ add_relative_time_idx=True,
+ add_target_scales=True,
+ add_encoder_length=True,
+ )
+
+ # create validation set (predict=True) which means to predict the last max_prediction_length points in time
+ # for each series
+ validation = TimeSeriesDataSet.from_dataset(
+ training, self.data, predict=True, stop_randomization=True
+ )
+
+ # create dataloaders for model
+ batch_size = kwargs.get("batch_size", 64)
+ train_dataloader = training.to_dataloader(
+ train=True, batch_size=batch_size, num_workers=0
+ )
+ val_dataloader = validation.to_dataloader(
+ train=False, batch_size=batch_size * 10, num_workers=0
+ )
+
+ return training, train_dataloader, val_dataloader
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ import warnings
+ import pytorch_lightning as pl
+ from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
+ import torch
+ from pytorch_forecasting import TemporalFusionTransformer
+ from pytorch_forecasting.metrics import QuantileLoss
+
+ warnings.filterwarnings("ignore")
+ current_time = time.time()
+ training, train_dataloader, val_dataloader = self.transform_ds(
+ X_train, y_train, **kwargs
+ )
+ params = self.params.copy()
+ gradient_clip_val = params.pop("gradient_clip_val")
+ params.pop("n_jobs")
+ max_epochs = kwargs.get("max_epochs", 20)
+ early_stop_callback = EarlyStopping(
+ monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min"
+ )
+
+ def _fit(log):
+ default_trainer_kwargs = dict(
+ gpus=kwargs.get("gpu_per_trial", [0])
+ if torch.cuda.is_available()
+ else None,
+ max_epochs=max_epochs,
+ gradient_clip_val=gradient_clip_val,
+ callbacks=[LearningRateMonitor(), early_stop_callback]
+ if log
+ else [early_stop_callback],
+ logger=log,
+ )
+ trainer = pl.Trainer(
+ **default_trainer_kwargs,
+ )
+ tft = TemporalFusionTransformer.from_dataset(
+ training,
+ **params,
+ lstm_layers=2, # 2 is mostly optimal according to documentation
+ output_size=7, # 7 quantiles by default
+ loss=QuantileLoss(),
+ log_interval=10 if log else 0,
+ # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches
+ reduce_on_plateau_patience=4,
+ )
+ # fit network
+ trainer.fit(
+ tft,
+ train_dataloaders=train_dataloader,
+ val_dataloaders=val_dataloader,
+ )
+ return trainer
+
+ # try:
+ # from pytorch_lightning.loggers import TensorBoardLogger
+
+ # logger = TensorBoardLogger(
+ # kwargs.get("log_dir", "lightning_logs")
+ # ) # logging results to a tensorboard
+ # trainer = _fit(log=logger)
+ # except ValueError:
+ # issue with pytorch forecasting model log_prediction() function
+ # pytorch-forecasting issue #1145
+ trainer = _fit(log=False)
+ best_model_path = trainer.checkpoint_callback.best_model_path
+ best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
+ train_time = time.time() - current_time
+ self._model = best_tft
+ return train_time
+
+ def predict(self, X):
+ import pandas as pd
+
+ ids = self.group_ids.copy()
+ ids.append(TS_TIMESTAMP_COL)
+ encoder_data = self.data[
+ lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length
+ ]
+ # following pytorchforecasting example, make all target values equal to the last data
+ last_data_cols = self.group_ids.copy()
+ last_data_cols.append(TS_VALUE_COL)
+ last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]
+ decoder_data = X
+ if "time_idx" not in decoder_data:
+ decoder_data = add_time_idx_col(decoder_data)
+ decoder_data["time_idx"] += (
+ encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()
+ )
+ # decoder_data[TS_VALUE_COL] = 0
+ decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)
+ decoder_data = decoder_data.sort_values(ids)
+ new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)
+ new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")
+ new_raw_predictions = self._model.predict(new_prediction_data)
+ index = [decoder_data[idx].to_numpy() for idx in ids]
+ predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)
+ return predictions
+
+
+class suppress_stdout_stderr(object):
+ def __init__(self):
+ # Open a pair of null files
+ self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]
+ # Save the actual stdout (1) and stderr (2) file descriptors.
+ self.save_fds = (os.dup(1), os.dup(2))
+
+ def __enter__(self):
+ # Assign the null pointers to stdout and stderr.
+ os.dup2(self.null_fds[0], 1)
+ os.dup2(self.null_fds[1], 2)
+
+ def __exit__(self, *_):
+ # Re-assign the real stdout/stderr back to (1) and (2)
+ os.dup2(self.save_fds[0], 1)
+ os.dup2(self.save_fds[1], 2)
+ # Close the null files
+ os.close(self.null_fds[0])
+ os.close(self.null_fds[1])
diff --git a/flaml/nlp/README.md b/flaml/automl/nlp/README.md
similarity index 100%
rename from flaml/nlp/README.md
rename to flaml/automl/nlp/README.md
diff --git a/flaml/nlp/__init__.py b/flaml/automl/nlp/__init__.py
similarity index 100%
rename from flaml/nlp/__init__.py
rename to flaml/automl/nlp/__init__.py
diff --git a/flaml/nlp/huggingface/__init__.py b/flaml/automl/nlp/huggingface/__init__.py
similarity index 100%
rename from flaml/nlp/huggingface/__init__.py
rename to flaml/automl/nlp/huggingface/__init__.py
diff --git a/flaml/nlp/huggingface/data_collator.py b/flaml/automl/nlp/huggingface/data_collator.py
similarity index 98%
rename from flaml/nlp/huggingface/data_collator.py
rename to flaml/automl/nlp/huggingface/data_collator.py
index 330846198..484e6a346 100644
--- a/flaml/nlp/huggingface/data_collator.py
+++ b/flaml/automl/nlp/huggingface/data_collator.py
@@ -6,7 +6,7 @@ from transformers.data.data_collator import (
)
from collections import OrderedDict
-from flaml.data import (
+from flaml.automl.data import (
TOKENCLASSIFICATION,
MULTICHOICECLASSIFICATION,
SUMMARIZATION,
diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/automl/nlp/huggingface/trainer.py
similarity index 100%
rename from flaml/nlp/huggingface/trainer.py
rename to flaml/automl/nlp/huggingface/trainer.py
diff --git a/flaml/nlp/huggingface/training_args.py b/flaml/automl/nlp/huggingface/training_args.py
similarity index 98%
rename from flaml/nlp/huggingface/training_args.py
rename to flaml/automl/nlp/huggingface/training_args.py
index c3eb839b6..b064f51f3 100644
--- a/flaml/nlp/huggingface/training_args.py
+++ b/flaml/automl/nlp/huggingface/training_args.py
@@ -1,7 +1,7 @@
import argparse
from dataclasses import dataclass, field
-from ...data import (
+from flaml.automl.data import (
NLG_TASKS,
)
from typing import Optional, List
@@ -33,7 +33,7 @@ class TrainingArgumentsForAuto(TrainingArguments):
When the task is sequence labeling/token classification, there are two formats of the labels:
(1) The token labels, i.e., [B-PER, I-PER, B-LOC]; (2) Id labels. For (2), need to pass the label_list (e.g., [B-PER, I-PER, B-LOC])
to convert the Id to token labels when computing the metric with metric_loss_score.
- See the example in [a simple token classification example](../../../Examples/AutoML-NLP#a-simple-token-classification-example).
+ See the example in [a simple token classification example](../../../../Examples/AutoML-NLP#a-simple-token-classification-example).
"""
task: str = field(default="seq-classification")
diff --git a/flaml/nlp/huggingface/utils.py b/flaml/automl/nlp/huggingface/utils.py
similarity index 99%
rename from flaml/nlp/huggingface/utils.py
rename to flaml/automl/nlp/huggingface/utils.py
index 13f1931fd..ba7558f09 100644
--- a/flaml/nlp/huggingface/utils.py
+++ b/flaml/automl/nlp/huggingface/utils.py
@@ -2,7 +2,7 @@ import pandas as pd
from itertools import chain
import numpy as np
-from ...data import (
+from flaml.automl.data import (
SUMMARIZATION,
SEQREGRESSION,
SEQCLASSIFICATION,
diff --git a/flaml/nlp/utils.py b/flaml/automl/nlp/utils.py
similarity index 98%
rename from flaml/nlp/utils.py
rename to flaml/automl/nlp/utils.py
index 5e69bead0..872c1d037 100644
--- a/flaml/nlp/utils.py
+++ b/flaml/automl/nlp/utils.py
@@ -1,13 +1,12 @@
from typing import Dict, Any
import numpy as np
-from ..data import (
+from flaml.automl.data import (
SUMMARIZATION,
SEQREGRESSION,
SEQCLASSIFICATION,
MULTICHOICECLASSIFICATION,
TOKENCLASSIFICATION,
- NLG_TASKS,
)
diff --git a/flaml/training_log.py b/flaml/automl/training_log.py
similarity index 100%
rename from flaml/training_log.py
rename to flaml/automl/training_log.py
diff --git a/flaml/data.py b/flaml/data.py
index 9deab1b79..522b47fe0 100644
--- a/flaml/data.py
+++ b/flaml/data.py
@@ -1,485 +1,9 @@
-# !
-# * Copyright (c) Microsoft Corporation. All rights reserved.
-# * Licensed under the MIT License. See LICENSE file in the
-# * project root for license information.
-import numpy as np
-from scipy.sparse import vstack, issparse
-import pandas as pd
-from pandas import DataFrame, Series
+import warnings
-from .training_log import training_log_reader
+from flaml.automl.data import *
-from datetime import datetime
-from typing import Dict, Union, List
-# TODO: if your task is not specified in here, define your task as an all-capitalized word
-SEQCLASSIFICATION = "seq-classification"
-MULTICHOICECLASSIFICATION = "multichoice-classification"
-TOKENCLASSIFICATION = "token-classification"
-CLASSIFICATION = (
- "binary",
- "multiclass",
- "classification",
- SEQCLASSIFICATION,
- MULTICHOICECLASSIFICATION,
- TOKENCLASSIFICATION,
+warnings.warn(
+ "Importing from `flaml.data` is deprecated. Please use `flaml.automl.data`.",
+ DeprecationWarning,
)
-SEQREGRESSION = "seq-regression"
-REGRESSION = ("regression", SEQREGRESSION)
-TS_FORECASTREGRESSION = (
- "forecast",
- "ts_forecast",
- "ts_forecast_regression",
-)
-TS_FORECASTCLASSIFICATION = "ts_forecast_classification"
-TS_FORECASTPANEL = "ts_forecast_panel"
-TS_FORECAST = (
- *TS_FORECASTREGRESSION,
- TS_FORECASTCLASSIFICATION,
- TS_FORECASTPANEL,
-)
-TS_TIMESTAMP_COL = "ds"
-TS_VALUE_COL = "y"
-SUMMARIZATION = "summarization"
-NLG_TASKS = (SUMMARIZATION,)
-NLU_TASKS = (
- SEQREGRESSION,
- SEQCLASSIFICATION,
- MULTICHOICECLASSIFICATION,
- TOKENCLASSIFICATION,
-)
-
-
-def _is_nlp_task(task):
- if task in NLU_TASKS or task in NLG_TASKS:
- return True
- else:
- return False
-
-
-def load_openml_dataset(
- dataset_id, data_dir=None, random_state=0, dataset_format="dataframe"
-):
- """Load dataset from open ML.
-
- If the file is not cached locally, download it from open ML.
-
- Args:
- dataset_id: An integer of the dataset id in openml.
- data_dir: A string of the path to store and load the data.
- random_state: An integer of the random seed for splitting data.
- dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'.
- Can choose from ['dataframe', 'array'].
- If 'dataframe', the returned dataset will be a Pandas DataFrame.
- If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix.
-
- Returns:
- X_train: Training data.
- X_test: Test data.
- y_train: A series or array of labels for training data.
- y_test: A series or array of labels for test data.
- """
- import os
- import openml
- import pickle
- from sklearn.model_selection import train_test_split
-
- filename = "openml_ds" + str(dataset_id) + ".pkl"
- filepath = os.path.join(data_dir, filename)
- if os.path.isfile(filepath):
- print("load dataset from", filepath)
- with open(filepath, "rb") as f:
- dataset = pickle.load(f)
- else:
- print("download dataset from openml")
- dataset = openml.datasets.get_dataset(dataset_id)
- if not os.path.exists(data_dir):
- os.makedirs(data_dir)
- with open(filepath, "wb") as f:
- pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
- print("Dataset name:", dataset.name)
- try:
- X, y, *__ = dataset.get_data(
- target=dataset.default_target_attribute, dataset_format=dataset_format
- )
- except ValueError:
- from sklearn.datasets import fetch_openml
-
- X, y = fetch_openml(data_id=dataset_id, return_X_y=True)
- X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state)
- print(
- "X_train.shape: {}, y_train.shape: {};\nX_test.shape: {}, y_test.shape: {}".format(
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
- )
- return X_train, X_test, y_train, y_test
-
-
-def load_openml_task(task_id, data_dir):
- """Load task from open ML.
-
- Use the first fold of the task.
- If the file is not cached locally, download it from open ML.
-
- Args:
- task_id: An integer of the task id in openml.
- data_dir: A string of the path to store and load the data.
-
- Returns:
- X_train: A dataframe of training data.
- X_test: A dataframe of test data.
- y_train: A series of labels for training data.
- y_test: A series of labels for test data.
- """
- import os
- import openml
- import pickle
-
- task = openml.tasks.get_task(task_id)
- filename = "openml_task" + str(task_id) + ".pkl"
- filepath = os.path.join(data_dir, filename)
- if os.path.isfile(filepath):
- print("load dataset from", filepath)
- with open(filepath, "rb") as f:
- dataset = pickle.load(f)
- else:
- print("download dataset from openml")
- dataset = task.get_dataset()
- with open(filepath, "wb") as f:
- pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
- X, y, _, _ = dataset.get_data(task.target_name)
- train_indices, test_indices = task.get_train_test_split_indices(
- repeat=0,
- fold=0,
- sample=0,
- )
- X_train = X.iloc[train_indices]
- y_train = y[train_indices]
- X_test = X.iloc[test_indices]
- y_test = y[test_indices]
- print(
- "X_train.shape: {}, y_train.shape: {},\nX_test.shape: {}, y_test.shape: {}".format(
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
- )
- return X_train, X_test, y_train, y_test
-
-
-def get_output_from_log(filename, time_budget):
- """Get output from log file.
-
- Args:
- filename: A string of the log file name.
- time_budget: A float of the time budget in seconds.
-
- Returns:
- search_time_list: A list of the finished time of each logged iter.
- best_error_list: A list of the best validation error after each logged iter.
- error_list: A list of the validation error of each logged iter.
- config_list: A list of the estimator, sample size and config of each logged iter.
- logged_metric_list: A list of the logged metric of each logged iter.
- """
-
- best_config = None
- best_learner = None
- best_val_loss = float("+inf")
-
- search_time_list = []
- config_list = []
- best_error_list = []
- error_list = []
- logged_metric_list = []
- best_config_list = []
- with training_log_reader(filename) as reader:
- for record in reader.records():
- time_used = record.wall_clock_time
- val_loss = record.validation_loss
- config = record.config
- learner = record.learner.split("_")[0]
- sample_size = record.sample_size
- metric = record.logged_metric
-
- if time_used < time_budget and np.isfinite(val_loss):
- if val_loss < best_val_loss:
- best_val_loss = val_loss
- best_config = config
- best_learner = learner
- best_config_list.append(best_config)
- search_time_list.append(time_used)
- best_error_list.append(best_val_loss)
- logged_metric_list.append(metric)
- error_list.append(val_loss)
- config_list.append(
- {
- "Current Learner": learner,
- "Current Sample": sample_size,
- "Current Hyper-parameters": record.config,
- "Best Learner": best_learner,
- "Best Hyper-parameters": best_config,
- }
- )
-
- return (
- search_time_list,
- best_error_list,
- error_list,
- config_list,
- logged_metric_list,
- )
-
-
-def concat(X1, X2):
- """concatenate two matrices vertically."""
- if isinstance(X1, (DataFrame, Series)):
- df = pd.concat([X1, X2], sort=False)
- df.reset_index(drop=True, inplace=True)
- if isinstance(X1, DataFrame):
- cat_columns = X1.select_dtypes(include="category").columns
- if len(cat_columns):
- df[cat_columns] = df[cat_columns].astype("category")
- return df
- if issparse(X1):
- return vstack((X1, X2))
- else:
- return np.concatenate([X1, X2])
-
-
-def add_time_idx_col(X):
- unique_dates = X[TS_TIMESTAMP_COL].drop_duplicates().sort_values(ascending=True)
- # assume no missing timestamps
- freq = pd.infer_freq(unique_dates)
- if freq == "MS":
- X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year * 12 + X[TS_TIMESTAMP_COL].dt.month
- elif freq == "Y":
- X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year
- else:
- # using time frequency to generate all time stamps and then indexing for time_idx
- # full_range = pd.date_range(X[TS_TIMESTAMP_COL].min(), X[TS_TIMESTAMP_COL].max(), freq=freq).to_list()
- # X["time_idx"] = [full_range.index(time) for time in X[TS_TIMESTAMP_COL]]
- # taking minimum difference in timestamp
- timestamps = unique_dates.view("int64")
- freq = int(timestamps.diff().mode())
- X["time_idx"] = timestamps - timestamps.min() / freq
- X["time_idx"] = X["time_idx"].astype("int")
- return X
-
-
-class DataTransformer:
- """Transform input training data."""
-
- def fit_transform(self, X: Union[DataFrame, np.array], y, task):
- """Fit transformer and process the input training data according to the task type.
-
- Args:
- X: A numpy array or a pandas dataframe of training data.
- y: A numpy array or a pandas series of labels.
- task: A string of the task type, e.g.,
- 'classification', 'regression', 'ts_forecast', 'rank'.
-
- Returns:
- X: Processed numpy array or pandas dataframe of training data.
- y: Processed numpy array or pandas series of labels.
- """
- if _is_nlp_task(task):
- # if the mode is NLP, check the type of input, each column must be either string or
- # ids (input ids, token type id, attention mask, etc.)
- str_columns = []
- for column in X.columns:
- if isinstance(X[column].iloc[0], str):
- str_columns.append(column)
- if len(str_columns) > 0:
- X[str_columns] = X[str_columns].astype("string")
- self._str_columns = str_columns
- elif isinstance(X, DataFrame):
- X = X.copy()
- n = X.shape[0]
- cat_columns, num_columns, datetime_columns = [], [], []
- drop = False
- if task in TS_FORECAST:
- X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
- if task is TS_FORECASTPANEL:
- if "time_idx" not in X:
- X = add_time_idx_col(X)
- ds_col = X.pop(TS_TIMESTAMP_COL)
- if isinstance(y, Series):
- y = y.rename(TS_VALUE_COL)
- for column in X.columns:
- # sklearn\utils\validation.py needs int/float values
- if X[column].dtype.name in ("object", "category"):
- if (
- X[column].nunique() == 1
- or X[column].nunique(dropna=True)
- == n - X[column].isnull().sum()
- ):
- X.drop(columns=column, inplace=True)
- drop = True
- elif X[column].dtype.name == "category":
- current_categories = X[column].cat.categories
- if "__NAN__" not in current_categories:
- X[column] = (
- X[column]
- .cat.add_categories("__NAN__")
- .fillna("__NAN__")
- )
- cat_columns.append(column)
- else:
- X[column] = X[column].fillna("__NAN__")
- cat_columns.append(column)
- elif X[column].nunique(dropna=True) < 2:
- X.drop(columns=column, inplace=True)
- drop = True
- else: # datetime or numeric
- if X[column].dtype.name == "datetime64[ns]":
- tmp_dt = X[column].dt
- new_columns_dict = {
- f"year_{column}": tmp_dt.year,
- f"month_{column}": tmp_dt.month,
- f"day_{column}": tmp_dt.day,
- f"hour_{column}": tmp_dt.hour,
- f"minute_{column}": tmp_dt.minute,
- f"second_{column}": tmp_dt.second,
- f"dayofweek_{column}": tmp_dt.dayofweek,
- f"dayofyear_{column}": tmp_dt.dayofyear,
- f"quarter_{column}": tmp_dt.quarter,
- }
- for key, value in new_columns_dict.items():
- if (
- key not in X.columns
- and value.nunique(dropna=False) >= 2
- ):
- X[key] = value
- num_columns.append(key)
- X[column] = X[column].map(datetime.toordinal)
- datetime_columns.append(column)
- del tmp_dt
- X[column] = X[column].fillna(np.nan)
- num_columns.append(column)
- X = X[cat_columns + num_columns]
- if task in TS_FORECAST:
- X.insert(0, TS_TIMESTAMP_COL, ds_col)
- if cat_columns:
- X[cat_columns] = X[cat_columns].astype("category")
- if num_columns:
- X_num = X[num_columns]
- if np.issubdtype(X_num.columns.dtype, np.integer) and (
- drop
- or min(X_num.columns) != 0
- or max(X_num.columns) != X_num.shape[1] - 1
- ):
- X_num.columns = range(X_num.shape[1])
- drop = True
- else:
- drop = False
- from sklearn.impute import SimpleImputer
- from sklearn.compose import ColumnTransformer
-
- self.transformer = ColumnTransformer(
- [
- (
- "continuous",
- SimpleImputer(missing_values=np.nan, strategy="median"),
- X_num.columns,
- )
- ]
- )
- X[num_columns] = self.transformer.fit_transform(X_num)
- self._cat_columns, self._num_columns, self._datetime_columns = (
- cat_columns,
- num_columns,
- datetime_columns,
- )
- self._drop = drop
- if (
- task in CLASSIFICATION
- or not pd.api.types.is_numeric_dtype(y)
- and task not in NLG_TASKS
- ):
- if task != TOKENCLASSIFICATION:
- from sklearn.preprocessing import LabelEncoder
-
- self.label_transformer = LabelEncoder()
- else:
- from .nlp.utils import LabelEncoderforTokenClassification
-
- self.label_transformer = LabelEncoderforTokenClassification()
- y = self.label_transformer.fit_transform(y)
-
- else:
- self.label_transformer = None
- self._task = task
- return X, y
-
- def transform(self, X: Union[DataFrame, np.array]):
- """Process data using fit transformer.
-
- Args:
- X: A numpy array or a pandas dataframe of training data.
-
- Returns:
- X: Processed numpy array or pandas dataframe of training data.
- """
- X = X.copy()
-
- if _is_nlp_task(self._task):
- # if the mode is NLP, check the type of input, each column must be either string or
- # ids (input ids, token type id, attention mask, etc.)
- if len(self._str_columns) > 0:
- X[self._str_columns] = X[self._str_columns].astype("string")
- elif isinstance(X, DataFrame):
- cat_columns, num_columns, datetime_columns = (
- self._cat_columns,
- self._num_columns,
- self._datetime_columns,
- )
- if self._task in TS_FORECAST:
- X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
- ds_col = X.pop(TS_TIMESTAMP_COL)
- for column in datetime_columns:
- tmp_dt = X[column].dt
- new_columns_dict = {
- f"year_{column}": tmp_dt.year,
- f"month_{column}": tmp_dt.month,
- f"day_{column}": tmp_dt.day,
- f"hour_{column}": tmp_dt.hour,
- f"minute_{column}": tmp_dt.minute,
- f"second_{column}": tmp_dt.second,
- f"dayofweek_{column}": tmp_dt.dayofweek,
- f"dayofyear_{column}": tmp_dt.dayofyear,
- f"quarter_{column}": tmp_dt.quarter,
- }
- for new_col_name, new_col_value in new_columns_dict.items():
- if new_col_name not in X.columns and new_col_name in num_columns:
- X[new_col_name] = new_col_value
- X[column] = X[column].map(datetime.toordinal)
- del tmp_dt
- X = X[cat_columns + num_columns].copy()
- if self._task in TS_FORECAST:
- X.insert(0, TS_TIMESTAMP_COL, ds_col)
- for column in cat_columns:
- if X[column].dtype.name == "object":
- X[column] = X[column].fillna("__NAN__")
- elif X[column].dtype.name == "category":
- current_categories = X[column].cat.categories
- if "__NAN__" not in current_categories:
- X[column] = (
- X[column].cat.add_categories("__NAN__").fillna("__NAN__")
- )
- if cat_columns:
- X[cat_columns] = X[cat_columns].astype("category")
- if num_columns:
- X_num = X[num_columns].fillna(np.nan)
- if self._drop:
- X_num.columns = range(X_num.shape[1])
- X[num_columns] = self.transformer.transform(X_num)
- return X
-
-
-def group_counts(groups):
- _, i, c = np.unique(groups, return_counts=True, return_index=True)
- return c[np.argsort(i)]
diff --git a/flaml/default/estimator.py b/flaml/default/estimator.py
index ca7c980da..f5eaf5875 100644
--- a/flaml/default/estimator.py
+++ b/flaml/default/estimator.py
@@ -1,6 +1,6 @@
import sklearn.ensemble as ensemble
from functools import wraps
-from flaml.data import CLASSIFICATION
+from flaml.automl.data import CLASSIFICATION
from .suggest import preprocess_and_suggest_hyperparams
DEFAULT_LOCATION = "default_location"
diff --git a/flaml/default/suggest.py b/flaml/default/suggest.py
index fbb02b069..71f4d74d8 100644
--- a/flaml/default/suggest.py
+++ b/flaml/default/suggest.py
@@ -3,8 +3,8 @@ from sklearn.neighbors import NearestNeighbors
import logging
import pathlib
import json
-from flaml.data import CLASSIFICATION, DataTransformer
-from flaml.ml import get_estimator_class, get_classification_objective
+from flaml.automl.data import CLASSIFICATION, DataTransformer
+from flaml.automl.ml import get_estimator_class, get_classification_objective
from flaml.version import __version__
LOCATION = pathlib.Path(__file__).parent.resolve()
diff --git a/flaml/ml.py b/flaml/ml.py
index ea7edfbd5..fcc3eb98a 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -1,749 +1,9 @@
-# !
-# * Copyright (c) FLAML authors. All rights reserved.
-# * Licensed under the MIT License. See LICENSE file in the
-# * project root for license information.
-import time
-import numpy as np
-import pandas as pd
-from sklearn.metrics import (
- mean_squared_error,
- r2_score,
- roc_auc_score,
- accuracy_score,
- mean_absolute_error,
- log_loss,
- average_precision_score,
- f1_score,
- mean_absolute_percentage_error,
- ndcg_score,
-)
-from sklearn.model_selection import RepeatedStratifiedKFold, GroupKFold, TimeSeriesSplit
-from .model import (
- XGBoostSklearnEstimator,
- XGBoost_TS,
- XGBoostLimitDepthEstimator,
- XGBoostLimitDepth_TS,
- RandomForestEstimator,
- RF_TS,
- LGBMEstimator,
- LGBM_TS,
- LRL1Classifier,
- LRL2Classifier,
- CatBoostEstimator,
- ExtraTreesEstimator,
- ExtraTrees_TS,
- KNeighborsEstimator,
- Prophet,
- ARIMA,
- SARIMAX,
- TransformersEstimator,
- TemporalFusionTransformerEstimator,
- TransformersEstimatorModelSelection,
-)
-from .data import CLASSIFICATION, group_counts, TS_FORECAST
-import logging
-
-logger = logging.getLogger(__name__)
-
-sklearn_metric_name_set = {
- "r2",
- "rmse",
- "mae",
- "mse",
- "accuracy",
- "roc_auc",
- "roc_auc_ovr",
- "roc_auc_ovo",
- "roc_auc_weighted",
- "roc_auc_ovr_weighted",
- "roc_auc_ovo_weighted",
- "log_loss",
- "mape",
- "f1",
- "ap",
- "ndcg",
- "micro_f1",
- "macro_f1",
-}
-huggingface_metric_to_mode = {
- "accuracy": "max",
- "bertscore": "max",
- "bleu": "max",
- "bleurt": "max",
- "cer": "min",
- "chrf": "min",
- "code_eval": "max",
- "comet": "max",
- "competition_math": "max",
- "coval": "max",
- "cuad": "max",
- "f1": "max",
- "gleu": "max",
- "google_bleu": "max",
- "matthews_correlation": "max",
- "meteor": "max",
- "pearsonr": "max",
- "precision": "max",
- "recall": "max",
- "rouge": "max",
- "sacrebleu": "max",
- "sari": "max",
- "seqeval": "max",
- "spearmanr": "max",
- "ter": "min",
- "wer": "min",
-}
-huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}
-
-
-def get_estimator_class(task, estimator_name):
- # when adding a new learner, need to add an elif branch
- if "xgboost" == estimator_name:
- estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator
- elif "xgb_limitdepth" == estimator_name:
- estimator_class = (
- XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator
- )
- elif "rf" == estimator_name:
- estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator
- elif "lgbm" == estimator_name:
- estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator
- elif "lrl1" == estimator_name:
- estimator_class = LRL1Classifier
- elif "lrl2" == estimator_name:
- estimator_class = LRL2Classifier
- elif "catboost" == estimator_name:
- estimator_class = CatBoostEstimator
- elif "extra_tree" == estimator_name:
- estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator
- elif "kneighbor" == estimator_name:
- estimator_class = KNeighborsEstimator
- elif "prophet" in estimator_name:
- estimator_class = Prophet
- elif estimator_name == "arima":
- estimator_class = ARIMA
- elif estimator_name == "sarimax":
- estimator_class = SARIMAX
- elif estimator_name == "transformer":
- estimator_class = TransformersEstimator
- elif estimator_name == "tft":
- estimator_class = TemporalFusionTransformerEstimator
- elif estimator_name == "transformer_ms":
- estimator_class = TransformersEstimatorModelSelection
- else:
- raise ValueError(
- estimator_name + " is not a built-in learner. "
- "Please use AutoML.add_learner() to add a customized learner."
- )
- return estimator_class
-
-
-def metric_loss_score(
- metric_name,
- y_processed_predict,
- y_processed_true,
- labels=None,
- sample_weight=None,
- groups=None,
-):
- # y_processed_predict and y_processed_true are processed id labels if the original were the token labels
- if is_in_sklearn_metric_name_set(metric_name):
- return sklearn_metric_loss_score(
- metric_name,
- y_processed_predict,
- y_processed_true,
- labels,
- sample_weight,
- groups,
- )
- else:
- try:
- import datasets
-
- datasets_metric_name = huggingface_submetric_to_metric.get(
- metric_name, metric_name.split(":")[0]
- )
- metric = datasets.load_metric(datasets_metric_name)
- metric_mode = huggingface_metric_to_mode[datasets_metric_name]
-
- if metric_name.startswith("seqeval"):
- y_processed_true = [
- [labels[tr] for tr in each_list] for each_list in y_processed_true
- ]
- elif metric in ("pearsonr", "spearmanr"):
- y_processed_true = (
- y_processed_true.to_list()
- if isinstance(y_processed_true, pd.Series)
- else list(y_processed_true)
- )
- score_dict = metric.compute(
- predictions=y_processed_predict, references=y_processed_true
- )
- if "rouge" in metric_name:
- score = score_dict[metric_name].mid.fmeasure
- elif metric_name.startswith("seqeval"):
- metric_submetric_names = metric_name.split(":")
- score = score_dict[
- metric_submetric_names[1]
- if len(metric_submetric_names) > 1
- else "overall_accuracy"
- ]
- else:
- score = score_dict[metric_name]
- except ImportError:
- raise ValueError(
- metric_name
- + " is not an built-in sklearn metric and nlp is not installed. "
- "Currently built-in sklearn metrics are: "
- "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"
- "log_loss, mape, f1, micro_f1, macro_f1, ap. "
- "If the metric is an nlp metric, please pip install flaml[nlp] ",
- "or pass a customized metric function to AutoML.fit(metric=func)",
- )
- # If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)
- # ask the user to provide a custom metric
- except FileNotFoundError:
- raise ValueError(
- metric_name + " is neither an sklearn metric nor a huggingface metric. "
- "Currently built-in sklearn metrics are: "
- "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"
- "log_loss, mape, f1, micro_f1, macro_f1, ap. "
- "Currently built-in huggingface metrics are: "
- + ", ".join(huggingface_metric_to_mode.keys())
- + ". Please pass a customized metric function to AutoML.fit(metric=func)"
- )
- if metric_mode == "max":
- return 1 - score
- else:
- return score
-
-
-def is_in_sklearn_metric_name_set(metric_name):
- return metric_name.startswith("ndcg") or metric_name in sklearn_metric_name_set
-
-
-def is_min_metric(metric_name):
- return (
- metric_name in ["rmse", "mae", "mse", "log_loss", "mape"]
- or huggingface_metric_to_mode.get(metric_name, None) == "min"
- )
-
-
-def sklearn_metric_loss_score(
- metric_name,
- y_predict,
- y_true,
- labels=None,
- sample_weight=None,
- groups=None,
-):
- """Loss using the specified metric.
-
- Args:
- metric_name: A string of the metric name, one of
- 'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'roc_auc_ovr',
- 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted',
- 'log_loss', 'mape', 'f1', 'ap', 'ndcg', 'micro_f1', 'macro_f1'.
- y_predict: A 1d or 2d numpy array of the predictions which can be
- used to calculate the metric. E.g., 2d for log_loss and 1d
- for others.
- y_true: A 1d numpy array of the true labels.
- labels: A list or an array of the unique labels.
- sample_weight: A 1d numpy array of the sample weight.
- groups: A 1d numpy array of the group labels.
-
- Returns:
- score: A float number of the loss, the lower the better.
- """
- metric_name = metric_name.lower()
-
- if "r2" == metric_name:
- score = 1.0 - r2_score(y_true, y_predict, sample_weight=sample_weight)
- elif metric_name == "rmse":
- score = np.sqrt(
- mean_squared_error(y_true, y_predict, sample_weight=sample_weight)
- )
- elif metric_name == "mae":
- score = mean_absolute_error(y_true, y_predict, sample_weight=sample_weight)
- elif metric_name == "mse":
- score = mean_squared_error(y_true, y_predict, sample_weight=sample_weight)
- elif metric_name == "accuracy":
- score = 1.0 - accuracy_score(y_true, y_predict, sample_weight=sample_weight)
- elif metric_name == "roc_auc":
- score = 1.0 - roc_auc_score(y_true, y_predict, sample_weight=sample_weight)
- elif metric_name == "roc_auc_ovr":
- score = 1.0 - roc_auc_score(
- y_true, y_predict, sample_weight=sample_weight, multi_class="ovr"
- )
- elif metric_name == "roc_auc_ovo":
- score = 1.0 - roc_auc_score(
- y_true, y_predict, sample_weight=sample_weight, multi_class="ovo"
- )
- elif metric_name == "roc_auc_weighted":
- score = 1.0 - roc_auc_score(
- y_true, y_predict, sample_weight=sample_weight, average="weighted"
- )
- elif metric_name == "roc_auc_ovo_weighted":
- score = 1.0 - roc_auc_score(
- y_true,
- y_predict,
- sample_weight=sample_weight,
- average="weighted",
- multi_class="ovo",
- )
- elif metric_name == "roc_auc_ovr_weighted":
- score = 1.0 - roc_auc_score(
- y_true,
- y_predict,
- sample_weight=sample_weight,
- average="weighted",
- multi_class="ovr",
- )
- elif "log_loss" == metric_name:
- score = log_loss(y_true, y_predict, labels=labels, sample_weight=sample_weight)
- elif "mape" == metric_name:
- try:
- score = mean_absolute_percentage_error(y_true, y_predict)
- except ValueError:
- return np.inf
- elif "micro_f1" == metric_name:
- score = 1 - f1_score(
- y_true, y_predict, sample_weight=sample_weight, average="micro"
- )
- elif "macro_f1" == metric_name:
- score = 1 - f1_score(
- y_true, y_predict, sample_weight=sample_weight, average="macro"
- )
- elif "f1" == metric_name:
- score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight)
- elif "ap" == metric_name:
- score = 1 - average_precision_score(
- y_true, y_predict, sample_weight=sample_weight
- )
- elif "ndcg" in metric_name:
- if "@" in metric_name:
- k = int(metric_name.split("@", 1)[-1])
- counts = group_counts(groups)
- score = 0
- psum = 0
- for c in counts:
- score -= ndcg_score(
- np.asarray([y_true[psum : psum + c]]),
- np.asarray([y_predict[psum : psum + c]]),
- k=k,
- )
- psum += c
- score /= len(counts)
- score += 1
- else:
- score = 1 - ndcg_score([y_true], [y_predict])
- return score
-
-
-def get_y_pred(estimator, X, eval_metric, obj):
- if eval_metric in ["roc_auc", "ap", "roc_auc_weighted"] and "binary" in obj:
- y_pred_classes = estimator.predict_proba(X)
- y_pred = y_pred_classes[:, 1] if y_pred_classes.ndim > 1 else y_pred_classes
- elif eval_metric in [
- "log_loss",
- "roc_auc",
- "roc_auc_ovr",
- "roc_auc_ovo",
- "roc_auc_ovo_weighted",
- "roc_auc_ovr_weighted",
- ]:
- y_pred = estimator.predict_proba(X)
- else:
- y_pred = estimator.predict(X)
- return y_pred
-
-
-def _eval_estimator(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- obj,
- labels=None,
- log_training_metric=False,
- fit_kwargs={},
-):
- if isinstance(eval_metric, str):
- pred_start = time.time()
- val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)
- pred_time = (time.time() - pred_start) / X_val.shape[0]
-
- val_loss = metric_loss_score(
- eval_metric,
- y_processed_predict=val_pred_y,
- y_processed_true=y_val,
- labels=labels,
- sample_weight=weight_val,
- groups=groups_val,
- )
- metric_for_logging = {"pred_time": pred_time}
- if log_training_metric:
- train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj)
- metric_for_logging["train_loss"] = metric_loss_score(
- eval_metric,
- train_pred_y,
- y_train,
- labels,
- fit_kwargs.get("sample_weight"),
- fit_kwargs.get("groups"),
- )
- else: # customized metric function
- val_loss, metric_for_logging = eval_metric(
- X_val,
- y_val,
- estimator,
- labels,
- X_train,
- y_train,
- weight_val,
- fit_kwargs.get("sample_weight"),
- config,
- groups_val,
- fit_kwargs.get("groups"),
- )
- pred_time = metric_for_logging.get("pred_time", 0)
- val_pred_y = None
- # eval_metric may return val_pred_y but not necessarily. Setting None for now.
- return val_loss, metric_for_logging, pred_time, val_pred_y
-
-
-def get_val_loss(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- obj,
- labels=None,
- budget=None,
- log_training_metric=False,
- fit_kwargs={},
- free_mem_ratio=0,
-):
-
- start = time.time()
- # if groups_val is not None:
- # fit_kwargs['groups_val'] = groups_val
- # fit_kwargs['X_val'] = X_val
- # fit_kwargs['y_val'] = y_val
- estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)
- val_loss, metric_for_logging, pred_time, _ = _eval_estimator(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- obj,
- labels,
- log_training_metric,
- fit_kwargs,
- )
- if hasattr(estimator, "intermediate_results"):
- metric_for_logging["intermediate_results"] = estimator.intermediate_results
- train_time = time.time() - start
- return val_loss, metric_for_logging, train_time, pred_time
-
-
-def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):
- metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)
- metrics_to_log = None
- for single_fold in log_metrics_folds:
- if metrics_to_log is None:
- metrics_to_log = single_fold
- elif isinstance(metrics_to_log, dict):
- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
- else:
- metrics_to_log += single_fold
- if metrics_to_log:
- n = len(val_loss_folds)
- metrics_to_log = (
- {k: v / n for k, v in metrics_to_log.items()}
- if isinstance(metrics_to_log, dict)
- else metrics_to_log / n
- )
- return metric_to_minimize, metrics_to_log
-
-
-def evaluate_model_CV(
- config,
- estimator,
- X_train_all,
- y_train_all,
- budget,
- kf,
- task,
- eval_metric,
- best_val_loss,
- cv_score_agg_func=None,
- log_training_metric=False,
- fit_kwargs={},
- free_mem_ratio=0,
-):
- if cv_score_agg_func is None:
- cv_score_agg_func = default_cv_score_agg_func
- start_time = time.time()
- val_loss_folds = []
- log_metric_folds = []
- metric = None
- train_time = pred_time = 0
- total_fold_num = 0
- n = kf.get_n_splits()
- X_train_split, y_train_split = X_train_all, y_train_all
- if task in CLASSIFICATION:
- labels = np.unique(y_train_all)
- else:
- labels = fit_kwargs.get(
- "label_list"
- ) # pass the label list on to compute the evaluation metric
- groups = None
- shuffle = getattr(kf, "shuffle", task not in TS_FORECAST)
- if isinstance(kf, RepeatedStratifiedKFold):
- kf = kf.split(X_train_split, y_train_split)
- elif isinstance(kf, GroupKFold):
- groups = kf.groups
- kf = kf.split(X_train_split, y_train_split, groups)
- shuffle = False
- elif isinstance(kf, TimeSeriesSplit):
- kf = kf.split(X_train_split, y_train_split)
- else:
- kf = kf.split(X_train_split)
- rng = np.random.RandomState(2020)
- budget_per_train = budget and budget / n
- if "sample_weight" in fit_kwargs:
- weight = fit_kwargs["sample_weight"]
- weight_val = None
- else:
- weight = weight_val = None
- for train_index, val_index in kf:
- if shuffle:
- train_index = rng.permutation(train_index)
- if isinstance(X_train_all, pd.DataFrame):
- X_train = X_train_split.iloc[train_index]
- X_val = X_train_split.iloc[val_index]
- else:
- X_train, X_val = X_train_split[train_index], X_train_split[val_index]
- y_train, y_val = y_train_split[train_index], y_train_split[val_index]
- estimator.cleanup()
- if weight is not None:
- fit_kwargs["sample_weight"], weight_val = (
- weight[train_index],
- weight[val_index],
- )
- if groups is not None:
- fit_kwargs["groups"] = groups[train_index]
- groups_val = groups[val_index]
- else:
- groups_val = None
- val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- task,
- labels,
- budget_per_train,
- log_training_metric=log_training_metric,
- fit_kwargs=fit_kwargs,
- free_mem_ratio=free_mem_ratio,
- )
- if isinstance(metric_i, dict) and "intermediate_results" in metric_i.keys():
- del metric_i["intermediate_results"]
- if weight is not None:
- fit_kwargs["sample_weight"] = weight
- total_fold_num += 1
- val_loss_folds.append(val_loss_i)
- log_metric_folds.append(metric_i)
- train_time += train_time_i
- pred_time += pred_time_i
- if budget and time.time() - start_time >= budget:
- break
- val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)
- n = total_fold_num
- pred_time /= n
- return val_loss, metric, train_time, pred_time
-
-
-def compute_estimator(
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- budget,
- kf,
- config_dic,
- task,
- estimator_name,
- eval_method,
- eval_metric,
- best_val_loss=np.Inf,
- n_jobs=1,
- estimator_class=None,
- cv_score_agg_func=None,
- log_training_metric=False,
- fit_kwargs={},
- free_mem_ratio=0,
-):
- estimator_class = estimator_class or get_estimator_class(task, estimator_name)
- estimator = estimator_class(
- **config_dic,
- task=task,
- n_jobs=n_jobs,
- )
-
- if isinstance(estimator, TransformersEstimator):
- # TODO: move the partial function to nlp
- fit_kwargs["metric"] = eval_metric
- fit_kwargs["X_val"] = X_val
- fit_kwargs["y_val"] = y_val
-
- if "holdout" == eval_method:
- val_loss, metric_for_logging, train_time, pred_time = get_val_loss(
- config_dic,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- task,
- labels=fit_kwargs.get(
- "label_list"
- ), # pass the label list on to compute the evaluation metric
- budget=budget,
- log_training_metric=log_training_metric,
- fit_kwargs=fit_kwargs,
- free_mem_ratio=0,
- )
- else:
- val_loss, metric_for_logging, train_time, pred_time = evaluate_model_CV(
- config_dic,
- estimator,
- X_train,
- y_train,
- budget,
- kf,
- task,
- eval_metric,
- best_val_loss,
- cv_score_agg_func,
- log_training_metric=log_training_metric,
- fit_kwargs=fit_kwargs,
- free_mem_ratio=0,
- )
-
- if isinstance(estimator, TransformersEstimator):
- del fit_kwargs["metric"], fit_kwargs["X_val"], fit_kwargs["y_val"]
-
- return estimator, val_loss, metric_for_logging, train_time, pred_time
-
-
-def train_estimator(
- config_dic,
- X_train,
- y_train,
- task,
- estimator_name,
- n_jobs=1,
- estimator_class=None,
- budget=None,
- fit_kwargs={},
- eval_metric=None,
- free_mem_ratio=0,
-):
- start_time = time.time()
- estimator_class = estimator_class or get_estimator_class(task, estimator_name)
- estimator = estimator_class(
- **config_dic,
- task=task,
- n_jobs=n_jobs,
- )
- if isinstance(estimator, TransformersEstimator):
- fit_kwargs["metric"] = eval_metric
-
- if X_train is not None:
- train_time = estimator.fit(
- X_train, y_train, budget, free_mem_ratio, **fit_kwargs
- )
- else:
- estimator = estimator.estimator_class(**estimator.params)
- train_time = time.time() - start_time
- return estimator, train_time
-
-
-def get_classification_objective(num_labels: int) -> str:
- if num_labels == 2:
- objective_name = "binary"
- else:
- objective_name = "multiclass"
- return objective_name
-
-
-def norm_confusion_matrix(y_true, y_pred):
- """normalized confusion matrix.
-
- Args:
- estimator: A multi-class classification estimator.
- y_true: A numpy array or a pandas series of true labels.
- y_pred: A numpy array or a pandas series of predicted labels.
-
- Returns:
- A normalized confusion matrix.
- """
- from sklearn.metrics import confusion_matrix
-
- conf_mat = confusion_matrix(y_true, y_pred)
- norm_conf_mat = conf_mat.astype("float") / conf_mat.sum(axis=1)[:, np.newaxis]
- return norm_conf_mat
-
-
-def multi_class_curves(y_true, y_pred_proba, curve_func):
- """Binarize the data for multi-class tasks and produce ROC or precision-recall curves.
-
- Args:
- y_true: A numpy array or a pandas series of true labels.
- y_pred_proba: A numpy array or a pandas dataframe of predicted probabilites.
- curve_func: A function to produce a curve (e.g., roc_curve or precision_recall_curve).
-
- Returns:
- A tuple of two dictionaries with the same set of keys (class indices).
- The first dictionary curve_x stores the x coordinates of each curve, e.g.,
- curve_x[0] is an 1D array of the x coordinates of class 0.
- The second dictionary curve_y stores the y coordinates of each curve, e.g.,
- curve_y[0] is an 1D array of the y coordinates of class 0.
- """
- from sklearn.preprocessing import label_binarize
-
- classes = np.unique(y_true)
- y_true_binary = label_binarize(y_true, classes=classes)
-
- curve_x, curve_y = {}, {}
- for i in range(len(classes)):
- curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])
- return curve_x, curve_y
+import warnings
+
+from flaml.automl.ml import *
+
+
+warnings.warn(
+ "Importing from `flaml.ml` is deprecated. Please use `flaml.automl.ml`.",
+ DeprecationWarning,
+)
diff --git a/flaml/model.py b/flaml/model.py
index 5fb2d723e..b780a67d1 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -1,2393 +1,9 @@
-# !
-# * Copyright (c) FLAML authors. All rights reserved.
-# * Licensed under the MIT License. See LICENSE file in the
-# * project root for license information.
-from contextlib import contextmanager
-from functools import partial
-import signal
-import os
-from typing import Callable, List
-import numpy as np
-import time
-from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
-from sklearn.ensemble import ExtraTreesRegressor, ExtraTreesClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.dummy import DummyClassifier, DummyRegressor
-from scipy.sparse import issparse
-import logging
-import shutil
-from pandas import DataFrame, Series, to_datetime
-import sys
-import math
-from . import tune
-from .data import (
- group_counts,
- CLASSIFICATION,
- add_time_idx_col,
- TS_FORECASTREGRESSION,
- TS_TIMESTAMP_COL,
- TS_VALUE_COL,
- SEQCLASSIFICATION,
- SEQREGRESSION,
- TOKENCLASSIFICATION,
- SUMMARIZATION,
- NLG_TASKS,
-)
-
-try:
- import psutil
-except ImportError:
- psutil = None
-try:
- import resource
-except ImportError:
- resource = None
-
-logger = logging.getLogger("flaml.automl")
-# FREE_MEM_RATIO = 0.2
-
-
-def TimeoutHandler(sig, frame):
- raise TimeoutError(sig, frame)
-
-
-@contextmanager
-def limit_resource(memory_limit, time_limit):
- if memory_limit > 0:
- soft, hard = resource.getrlimit(resource.RLIMIT_AS)
- if soft < 0 and (hard < 0 or memory_limit <= hard) or memory_limit < soft:
- try:
- resource.setrlimit(resource.RLIMIT_AS, (int(memory_limit), hard))
- except ValueError:
- # According to https://bugs.python.org/issue40518, it's a mac-specific error.
- pass
- main_thread = False
- if time_limit is not None:
- try:
- signal.signal(signal.SIGALRM, TimeoutHandler)
- signal.alarm(int(time_limit) or 1)
- main_thread = True
- except ValueError:
- pass
- try:
- yield
- finally:
- if main_thread:
- signal.alarm(0)
- if memory_limit > 0:
- resource.setrlimit(resource.RLIMIT_AS, (soft, hard))
-
-
-class BaseEstimator:
- """The abstract class for all learners.
-
- Typical examples:
- * XGBoostEstimator: for regression.
- * XGBoostSklearnEstimator: for classification.
- * LGBMEstimator, RandomForestEstimator, LRL1Classifier, LRL2Classifier:
- for both regression and classification.
- """
-
- def __init__(self, task="binary", **config):
- """Constructor.
-
- Args:
- task: A string of the task type, one of
- 'binary', 'multiclass', 'regression', 'rank', 'seq-classification',
- 'seq-regression', 'token-classification', 'multichoice-classification',
- 'summarization', 'ts_forecast', 'ts_forecast_classification'.
- config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.
- n_jobs is the number of parallel threads.
- """
- self._task = task
- self.params = self.config2params(config)
- self.estimator_class = self._model = None
- if "_estimator_type" in config:
- self._estimator_type = self.params.pop("_estimator_type")
- else:
- self._estimator_type = (
- "classifier" if task in CLASSIFICATION else "regressor"
- )
-
- def get_params(self, deep=False):
- params = self.params.copy()
- params["task"] = self._task
- if hasattr(self, "_estimator_type"):
- params["_estimator_type"] = self._estimator_type
- return params
-
- @property
- def classes_(self):
- return self._model.classes_
-
- @property
- def n_features_in_(self):
- return self._model.n_features_in_
-
- @property
- def model(self):
- """Trained model after fit() is called, or None before fit() is called."""
- return self._model
-
- @property
- def estimator(self):
- """Trained model after fit() is called, or None before fit() is called."""
- return self._model
-
- @property
- def feature_names_in_(self):
- """
- if self._model has attribute feature_names_in_, return it.
- otherwise, if self._model has attribute feature_name_, return it.
- otherwise, if self._model has attribute feature_names, return it.
- otherwise, if self._model has method get_booster, return the feature names.
- otherwise, return None.
- """
- if hasattr(self._model, "feature_names_in_"): # for sklearn, xgboost>=1.6
- return self._model.feature_names_in_
- if hasattr(self._model, "feature_name_"): # for lightgbm
- return self._model.feature_name_
- if hasattr(self._model, "feature_names"): # for XGBoostEstimator
- return self._model.feature_names
- if hasattr(self._model, "get_booster"):
- # get feature names for xgboost<1.6
- # https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.Booster.feature_names
- booster = self._model.get_booster()
- return booster.feature_names
- return None
-
- @property
- def feature_importances_(self):
- """
- if self._model has attribute feature_importances_, return it.
- otherwise, if self._model has attribute coef_, return it.
- otherwise, return None.
- """
- if hasattr(self._model, "feature_importances_"):
- # for sklearn, lightgbm, catboost, xgboost
- return self._model.feature_importances_
- elif hasattr(self._model, "coef_"): # for linear models
- return self._model.coef_
- else:
- return None
-
- def _preprocess(self, X):
- return X
-
- def _fit(self, X_train, y_train, **kwargs):
-
- current_time = time.time()
- if "groups" in kwargs:
- kwargs = kwargs.copy()
- groups = kwargs.pop("groups")
- if self._task == "rank":
- kwargs["group"] = group_counts(groups)
- # groups_val = kwargs.get('groups_val')
- # if groups_val is not None:
- # kwargs['eval_group'] = [group_counts(groups_val)]
- # kwargs['eval_set'] = [
- # (kwargs['X_val'], kwargs['y_val'])]
- # kwargs['verbose'] = False
- # del kwargs['groups_val'], kwargs['X_val'], kwargs['y_val']
- X_train = self._preprocess(X_train)
- model = self.estimator_class(**self.params)
- if logger.level == logging.DEBUG:
- # xgboost 1.6 doesn't display all the params in the model str
- logger.debug(f"flaml.model - {model} fit started with params {self.params}")
- model.fit(X_train, y_train, **kwargs)
- if logger.level == logging.DEBUG:
- logger.debug(f"flaml.model - {model} fit finished")
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- """Train the model from given training data.
-
- Args:
- X_train: A numpy array or a dataframe of training data in shape n*m.
- y_train: A numpy array or a series of labels in shape n*1.
- budget: A float of the time budget in seconds.
- free_mem_ratio: A float between 0 and 1 for the free memory ratio to keep during training.
-
- Returns:
- train_time: A float of the training time in seconds.
- """
- if (
- getattr(self, "limit_resource", None)
- and resource is not None
- and (budget is not None or psutil is not None)
- ):
- start_time = time.time()
- mem = psutil.virtual_memory() if psutil is not None else None
- try:
- with limit_resource(
- mem.available * (1 - free_mem_ratio)
- + psutil.Process(os.getpid()).memory_info().rss
- if mem is not None
- else -1,
- budget,
- ):
- train_time = self._fit(X_train, y_train, **kwargs)
- except (MemoryError, TimeoutError) as e:
- logger.warning(f"{e.__class__} {e}")
- if self._task in CLASSIFICATION:
- model = DummyClassifier()
- else:
- model = DummyRegressor()
- X_train = self._preprocess(X_train)
- model.fit(X_train, y_train)
- self._model = model
- train_time = time.time() - start_time
- else:
- train_time = self._fit(X_train, y_train, **kwargs)
- return train_time
-
- def predict(self, X, **kwargs):
- """Predict label from features.
-
- Args:
- X: A numpy array or a dataframe of featurized instances, shape n*m.
-
- Returns:
- A numpy array of shape n*1.
- Each element is the label for a instance.
- """
- if self._model is not None:
- X = self._preprocess(X)
- return self._model.predict(X, **kwargs)
- else:
- logger.warning(
- "Estimator is not fit yet. Please run fit() before predict()."
- )
- return np.ones(X.shape[0])
-
- def predict_proba(self, X, **kwargs):
- """Predict the probability of each class from features.
-
- Only works for classification problems
-
- Args:
- X: A numpy array of featurized instances, shape n*m.
-
- Returns:
- A numpy array of shape n*c. c is the # classes.
- Each element at (i,j) is the probability for instance i to be in
- class j.
- """
- assert self._task in CLASSIFICATION, "predict_proba() only for classification."
-
- X = self._preprocess(X)
- return self._model.predict_proba(X, **kwargs)
-
- def score(self, X_val: DataFrame, y_val: Series, **kwargs):
- """Report the evaluation score of a trained estimator.
-
-
- Args:
- X_val: A pandas dataframe of the validation input data.
- y_val: A pandas series of the validation label.
- kwargs: keyword argument of the evaluation function, for example:
- - metric: A string of the metric name or a function
- e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',
- 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',
- 'mape'. Default is 'auto'.
- If metric is given, the score will report the user specified metric.
- If metric is not given, the metric is set to accuracy for classification and r2
- for regression.
- You can also pass a customized metric function, for examples on how to pass a
- customized metric function, please check
- [test/nlp/test_autohf_custom_metric.py](https://github.com/microsoft/FLAML/blob/main/test/nlp/test_autohf_custom_metric.py) and
- [test/automl/test_multiclass.py](https://github.com/microsoft/FLAML/blob/main/test/automl/test_multiclass.py).
-
- Returns:
- The evaluation score on the validation dataset.
- """
- from .ml import metric_loss_score
- from .ml import is_min_metric
-
- if self._model is not None:
- if self._task == "rank":
- raise NotImplementedError(
- "AutoML.score() is not implemented for ranking"
- )
- else:
- X_val = self._preprocess(X_val)
- metric = kwargs.pop("metric", None)
- if metric:
- y_pred = self.predict(X_val, **kwargs)
- if is_min_metric(metric):
- return metric_loss_score(metric, y_pred, y_val)
- else:
- return 1.0 - metric_loss_score(metric, y_pred, y_val)
- else:
- return self._model.score(X_val, y_val, **kwargs)
- else:
- logger.warning(
- "Estimator is not fit yet. Please run fit() before predict()."
- )
- return 0.0
-
- def cleanup(self):
- del self._model
- self._model = None
-
- @classmethod
- def search_space(cls, data_size, task, **params):
- """[required method] search space.
-
- Args:
- data_size: A tuple of two integers, number of rows and columns.
- task: A str of the task type, e.g., "binary", "multiclass", "regression".
-
- Returns:
- A dictionary of the search space.
- Each key is the name of a hyperparameter, and value is a dict with
- its domain (required) and low_cost_init_value, init_value,
- cat_hp_cost (if applicable).
- e.g., ```{'domain': tune.randint(lower=1, upper=10), 'init_value': 1}```.
- """
- return {}
-
- @classmethod
- def size(cls, config: dict) -> float:
- """[optional method] memory size of the estimator in bytes.
-
- Args:
- config: A dict of the hyperparameter config.
-
- Returns:
- A float of the memory size required by the estimator to train the
- given config.
- """
- return 1.0
-
- @classmethod
- def cost_relative2lgbm(cls) -> float:
- """[optional method] relative cost compared to lightgbm."""
- return 1.0
-
- @classmethod
- def init(cls):
- """[optional method] initialize the class."""
- pass
-
- def config2params(self, config: dict) -> dict:
- """[optional method] config dict to params dict
-
- Args:
- config: A dict of the hyperparameter config.
-
- Returns:
- A dict that will be passed to self.estimator_class's constructor.
- """
- params = config.copy()
- if "FLAML_sample_size" in params:
- params.pop("FLAML_sample_size")
- return params
-
-
-class TransformersEstimator(BaseEstimator):
- """The class for fine-tuning language models, using huggingface transformers API."""
-
- ITER_HP = "global_max_steps"
-
- def __init__(self, task="seq-classification", **config):
- super().__init__(task, **config)
- import uuid
-
- self.trial_id = str(uuid.uuid1().hex)[:8]
- if task not in NLG_TASKS: # TODO: not in NLG_TASKS
- from .nlp.huggingface.training_args import (
- TrainingArgumentsForAuto as TrainingArguments,
- )
- else:
- from .nlp.huggingface.training_args import (
- Seq2SeqTrainingArgumentsForAuto as TrainingArguments,
- )
- self._TrainingArguments = TrainingArguments
-
- @classmethod
- def search_space(cls, data_size, task, **params):
- search_space_dict = {
- "learning_rate": {
- "domain": tune.loguniform(1e-6, 1e-4),
- "init_value": 1e-5,
- },
- "num_train_epochs": {
- "domain": tune.choice([1, 2, 3, 4, 5]),
- "init_value": 3, # to be consistent with roberta
- "low_cost_init_value": 1,
- },
- "per_device_train_batch_size": {
- "domain": tune.choice([4, 8, 16, 32, 64]),
- "init_value": 32,
- "low_cost_init_value": 64,
- },
- "seed": {
- "domain": tune.choice(range(1, 40)),
- "init_value": 20,
- },
- "global_max_steps": {
- "domain": sys.maxsize,
- "init_value": sys.maxsize,
- },
- }
-
- return search_space_dict
-
- @property
- def fp16(self):
- return self._kwargs.get("gpu_per_trial") and self._training_args.fp16
-
- @property
- def no_cuda(self):
- return not self._kwargs.get("gpu_per_trial")
-
- def _set_training_args(self, **kwargs):
- from .nlp.utils import date_str, Counter
-
- for (key, val) in kwargs.items():
- assert key not in self.params, (
- "Since {} is in the search space, it cannot exist in 'custom_fit_kwargs' at the same time."
- "If you need to fix the value of {} to {}, the only way is to add a single-value domain in the search "
- "space by adding:\n '{}': {{ 'domain': {} }} to 'custom_hp'. For example:"
- 'automl_settings["custom_hp"] = {{ "transformer": {{ "model_path": {{ "domain" : '
- '"google/electra-small-discriminator" }} }} }}'.format(
- key, key, val, key, val
- )
- )
-
- """
- If use has specified any custom args for TrainingArguments, update these arguments
- """
- self._training_args = self._TrainingArguments(**kwargs)
-
- """
- Update the attributes in TrainingArguments with self.params values
- """
- for key, val in self.params.items():
- if hasattr(self._training_args, key):
- setattr(self._training_args, key, val)
-
- """
- Update the attributes in TrainingArguments that depends on the values of self.params
- """
- local_dir = os.path.join(
- self._training_args.output_dir, "train_{}".format(date_str())
- )
- if self._use_ray is True:
- import ray
-
- self._training_args.output_dir = ray.tune.get_trial_dir()
- else:
- self._training_args.output_dir = Counter.get_trial_fold_name(
- local_dir, self.params, self.trial_id
- )
-
- self._training_args.fp16 = self.fp16
- self._training_args.no_cuda = self.no_cuda
-
- if (
- self._task == TOKENCLASSIFICATION
- and self._training_args.max_seq_length is not None
- ):
- logger.warning(
- "For token classification task, FLAML currently does not support customizing the max_seq_length, max_seq_length will be reset to None."
- )
- setattr(self._training_args, "max_seq_length", None)
-
- def _tokenize_text(self, X, y=None, **kwargs):
- from .nlp.huggingface.utils import tokenize_text
- from .nlp.utils import is_a_list_of_str
-
- is_str = str(X.dtypes[0]) in ("string", "str")
- is_list_of_str = is_a_list_of_str(X[list(X.keys())[0]].to_list()[0])
-
- if is_str or is_list_of_str:
- return tokenize_text(
- X=X,
- Y=y,
- task=self._task,
- hf_args=self._training_args,
- tokenizer=self.tokenizer,
- )
- else:
- return X, y
-
- def _model_init(self):
- from .nlp.huggingface.utils import load_model
-
- this_model = load_model(
- checkpoint_path=self._training_args.model_path,
- task=self._task,
- num_labels=self.num_labels,
- )
- return this_model
-
- def _preprocess_data(self, X, y):
- from datasets import Dataset
-
- processed_X, processed_y_df = self._tokenize_text(X=X, y=y, **self._kwargs)
- # convert y from pd.DataFrame back to pd.Series
- processed_y = processed_y_df.iloc[:, 0]
-
- processed_dataset = Dataset.from_pandas(processed_X.join(processed_y_df))
-
- return processed_dataset, processed_X, processed_y
-
- @property
- def num_labels(self):
- if self._task == SEQREGRESSION:
- return 1
- elif self._task == SEQCLASSIFICATION:
- return len(set(self._y_train))
- elif self._task == TOKENCLASSIFICATION:
- return len(self._training_args.label_list)
- else:
- return None
-
- @property
- def tokenizer(self):
- from transformers import AutoTokenizer
-
- if self._task == SUMMARIZATION:
- return AutoTokenizer.from_pretrained(
- pretrained_model_name_or_path=self._training_args.model_path,
- cache_dir=None,
- use_fast=True,
- revision="main",
- use_auth_token=None,
- )
- else:
- return AutoTokenizer.from_pretrained(
- self._training_args.model_path,
- use_fast=True,
- add_prefix_space=self._add_prefix_space,
- )
-
- @property
- def data_collator(self):
- from .nlp.huggingface.data_collator import task_to_datacollator_class
-
- data_collator_class = task_to_datacollator_class.get(self._task)
-
- if data_collator_class:
- kwargs = {
- "model": self._model_init(),
- # need to set model, or there's ValueError: Expected input batch_size (..) to match target batch_size (..)
- "label_pad_token_id": -100, # pad with token id -100
- "pad_to_multiple_of": 8,
- # pad to multiple of 8 because quote Transformers: "This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta)"
- "tokenizer": self.tokenizer,
- }
-
- for key in list(kwargs.keys()):
- if (
- key not in data_collator_class.__dict__.keys()
- and key != "tokenizer"
- ):
- del kwargs[key]
- return data_collator_class(**kwargs)
- else:
- return None
-
- def fit(
- self,
- X_train: DataFrame,
- y_train: Series,
- budget=None,
- free_mem_ratio=0,
- X_val=None,
- y_val=None,
- gpu_per_trial=None,
- metric=None,
- **kwargs,
- ):
- import transformers
-
- transformers.logging.set_verbosity_error()
-
- from transformers import TrainerCallback
- from transformers.trainer_utils import set_seed
- from .nlp.huggingface.trainer import TrainerForAuto
-
- try:
- from ray.tune import is_session_enabled
-
- self._use_ray = is_session_enabled()
- except ImportError:
- self._use_ray = False
-
- this_params = self.params
- self._kwargs = kwargs
-
- self._X_train, self._y_train = X_train, y_train
- self._set_training_args(**kwargs)
- self._add_prefix_space = (
- "roberta" in self._training_args.model_path
- ) # If using roberta model, must set add_prefix_space to True to avoid the assertion error at
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249
-
- train_dataset, self._X_train, self._y_train = self._preprocess_data(
- X_train, y_train
- )
- if X_val is not None:
- eval_dataset, self._X_val, self._y_val = self._preprocess_data(X_val, y_val)
- else:
- eval_dataset, self._X_val, self._y_val = None, None, None
-
- set_seed(self.params.get("seed", self._training_args.seed))
- self._metric = metric
-
- class EarlyStoppingCallbackForAuto(TrainerCallback):
- def on_train_begin(self, args, state, control, **callback_kwargs):
- self.train_begin_time = time.time()
-
- def on_step_begin(self, args, state, control, **callback_kwargs):
- self.step_begin_time = time.time()
-
- def on_step_end(self, args, state, control, **callback_kwargs):
- if state.global_step == 1:
- self.time_per_iter = time.time() - self.step_begin_time
- if (
- budget
- and (
- time.time() + self.time_per_iter
- > self.train_begin_time + budget
- )
- or state.global_step >= this_params[TransformersEstimator.ITER_HP]
- ):
- control.should_training_stop = True
- control.should_save = True
- control.should_evaluate = True
- return control
-
- def on_epoch_end(self, args, state, control, **callback_kwargs):
- if (
- control.should_training_stop
- or state.epoch + 1 >= args.num_train_epochs
- ):
- control.should_save = True
- control.should_evaluate = True
-
- self._trainer = TrainerForAuto(
- args=self._training_args,
- model_init=self._model_init,
- train_dataset=train_dataset,
- eval_dataset=eval_dataset,
- tokenizer=self.tokenizer,
- data_collator=self.data_collator,
- compute_metrics=self._compute_metrics_by_dataset_name,
- callbacks=[EarlyStoppingCallbackForAuto],
- )
-
- if self._task in NLG_TASKS:
- setattr(self._trainer, "_is_seq2seq", True)
-
- """
- When not using ray for tuning, set the limit of CUDA_VISIBLE_DEVICES to math.ceil(gpu_per_trial),
- so each estimator does not see all the GPUs
- """
- if gpu_per_trial is not None:
- tmp_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "")
- self._trainer.args._n_gpu = gpu_per_trial
-
- # if gpu_per_trial == 0:
- # os.environ["CUDA_VISIBLE_DEVICES"] = ""
- if tmp_cuda_visible_devices.count(",") != math.ceil(gpu_per_trial) - 1:
- os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(
- [str(x) for x in range(math.ceil(gpu_per_trial))]
- )
-
- import time
-
- start_time = time.time()
- self._trainer.train()
-
- if gpu_per_trial is not None:
- os.environ["CUDA_VISIBLE_DEVICES"] = tmp_cuda_visible_devices
-
- self.params[self.ITER_HP] = self._trainer.state.global_step
-
- self._checkpoint_path = self._select_checkpoint(self._trainer)
- self._ckpt_remains = list(self._trainer.ckpt_to_metric.keys())
-
- if hasattr(self._trainer, "intermediate_results"):
- self.intermediate_results = [
- x[1]
- for x in sorted(
- self._trainer.intermediate_results.items(), key=lambda x: x[0]
- )
- ]
- self._trainer = None
-
- return time.time() - start_time
-
- def _delete_one_ckpt(self, ckpt_location):
- if self._use_ray is False:
- try:
- shutil.rmtree(ckpt_location)
- except FileNotFoundError:
- logger.warning("checkpoint {} not found".format(ckpt_location))
-
- def cleanup(self):
- super().cleanup()
- if hasattr(self, "_ckpt_remains"):
- for each_ckpt in self._ckpt_remains:
- self._delete_one_ckpt(each_ckpt)
-
- def _select_checkpoint(self, trainer):
- from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
-
- if trainer.ckpt_to_metric:
- best_ckpt, _ = min(
- trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_automl_metric"]
- )
- best_ckpt_global_step = trainer.ckpt_to_global_step[best_ckpt]
- for each_ckpt in list(trainer.ckpt_to_metric):
- if each_ckpt != best_ckpt:
- del trainer.ckpt_to_metric[each_ckpt]
- del trainer.ckpt_to_global_step[each_ckpt]
- self._delete_one_ckpt(each_ckpt)
- else:
- best_ckpt_global_step = trainer.state.global_step
- best_ckpt = os.path.join(
- trainer.args.output_dir,
- f"{PREFIX_CHECKPOINT_DIR}-{best_ckpt_global_step}",
- )
- self.params[self.ITER_HP] = best_ckpt_global_step
- logger.debug(trainer.state.global_step)
- logger.debug(trainer.ckpt_to_global_step)
- return best_ckpt
-
- def _compute_metrics_by_dataset_name(self, eval_pred):
- # TODO: call self._metric(eval_pred, self)
- if isinstance(self._metric, str):
- from .ml import metric_loss_score
- from .nlp.huggingface.utils import postprocess_prediction_and_true
-
- predictions, y_true = eval_pred
- # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text
- processed_predictions, processed_y_true = postprocess_prediction_and_true(
- task=self._task,
- y_pred=predictions,
- tokenizer=self.tokenizer,
- hf_args=self._training_args,
- y_true=y_true,
- )
- metric_dict = {
- "automl_metric": metric_loss_score(
- metric_name=self._metric,
- y_processed_predict=processed_predictions,
- y_processed_true=processed_y_true,
- labels=self._training_args.label_list,
- )
- }
- else:
- # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)
- loss, metric_dict = self._metric(
- X_test=self._X_val,
- y_test=self._y_val,
- estimator=self,
- labels=None,
- X_train=self._X_train,
- y_train=self._y_train,
- )
- metric_dict["automl_metric"] = loss
-
- return metric_dict
-
- def _init_model_for_predict(self):
- from .nlp.huggingface.trainer import TrainerForAuto
-
- """
- Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistent
- with HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947
- """
- training_args = self._TrainingArguments(
- local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16
- )
- for key, val in self._training_args.__dict__.items():
- if key not in ("local_rank", "model_path", "fp16"):
- setattr(training_args, key, val)
- self._training_args = training_args
-
- new_trainer = TrainerForAuto(
- model=self._model_init(),
- args=self._training_args,
- data_collator=self.data_collator,
- compute_metrics=self._compute_metrics_by_dataset_name,
- )
- if self._task in NLG_TASKS:
- setattr(new_trainer, "_is_seq2seq", True)
- return new_trainer
-
- def predict_proba(self, X, **pred_kwargs):
- from datasets import Dataset
-
- if pred_kwargs:
- for key, val in pred_kwargs.items():
- setattr(self._training_args, key, val)
-
- assert (
- self._task in CLASSIFICATION
- ), "predict_proba() only for classification tasks."
-
- X_test, _ = self._tokenize_text(X, **self._kwargs)
- test_dataset = Dataset.from_pandas(X_test)
-
- new_trainer = self._init_model_for_predict()
- predictions = new_trainer.predict(test_dataset)
- return predictions.predictions
-
- def score(self, X_val: DataFrame, y_val: Series, **kwargs):
- import transformers
-
- transformers.logging.set_verbosity_error()
-
- self._metric = kwargs["metric"]
-
- eval_dataset, X_val, y_val = self._preprocess_data(X_val, y_val)
-
- new_trainer = self._init_model_for_predict()
- return new_trainer.evaluate(eval_dataset)
-
- def predict(self, X, **pred_kwargs):
- import transformers
- from datasets import Dataset
- from .nlp.huggingface.utils import postprocess_prediction_and_true
-
- transformers.logging.set_verbosity_error()
-
- if pred_kwargs:
- for key, val in pred_kwargs.items():
- setattr(self._training_args, key, val)
-
- X_test, _ = self._tokenize_text(X, **self._kwargs)
- test_dataset = Dataset.from_pandas(X_test)
-
- new_trainer = self._init_model_for_predict()
-
- if self._task not in NLG_TASKS:
- predictions = new_trainer.predict(test_dataset)
- else:
- predictions = new_trainer.predict(
- test_dataset,
- metric_key_prefix="predict",
- )
- post_y_pred, _ = postprocess_prediction_and_true(
- task=self._task,
- y_pred=predictions.predictions,
- tokenizer=self.tokenizer,
- hf_args=self._training_args,
- X=X,
- )
- return post_y_pred
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- params[TransformersEstimator.ITER_HP] = params.get(
- TransformersEstimator.ITER_HP, sys.maxsize
- )
- return params
-
-
-class TransformersEstimatorModelSelection(TransformersEstimator):
- def __init__(self, task="seq-classification", **config):
- super().__init__(task, **config)
-
- @classmethod
- def search_space(cls, data_size, task, **params):
- search_space_dict = TransformersEstimator.search_space(
- data_size, task, **params
- )
-
- """
- For model selection, use the same search space regardless of memory constraint
- If OOM, user should change the search space themselves
- """
-
- search_space_dict["model_path"] = {
- "domain": tune.choice(
- [
- "google/electra-base-discriminator",
- "bert-base-uncased",
- "roberta-base",
- "facebook/muppet-roberta-base",
- "google/electra-small-discriminator",
- ]
- ),
- "init_value": "facebook/muppet-roberta-base",
- }
- return search_space_dict
-
-
-class SKLearnEstimator(BaseEstimator):
- """
- The base class for tuning scikit-learn estimators.
-
- Subclasses can modify the function signature of ``__init__`` to
- ignore the values in ``config`` that are not relevant to the constructor
- of their underlying estimator. For example, some regressors in ``scikit-learn``
- don't accept the ``n_jobs`` parameter contained in ``config``. For these,
- one can add ``n_jobs=None,`` before ``**config`` to make sure ``config`` doesn't
- contain an ``n_jobs`` key.
- """
-
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
-
- def _preprocess(self, X):
- if isinstance(X, DataFrame):
- cat_columns = X.select_dtypes(include=["category"]).columns
- if not cat_columns.empty:
- X = X.copy()
- X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)
- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
- # numpy array is not of numeric dtype
- X = DataFrame(X)
- for col in X.columns:
- if isinstance(X[col][0], str):
- X[col] = X[col].astype("category").cat.codes
- X = X.to_numpy()
- return X
-
-
-class LGBMEstimator(BaseEstimator):
- """The class for tuning LGBM, using sklearn API."""
-
- ITER_HP = "n_estimators"
- HAS_CALLBACK = True
- DEFAULT_ITER = 100
-
- @classmethod
- def search_space(cls, data_size, **params):
- upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower
- return {
- "n_estimators": {
- "domain": tune.lograndint(lower=4, upper=upper),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- "num_leaves": {
- "domain": tune.lograndint(lower=4, upper=upper),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- "min_child_samples": {
- "domain": tune.lograndint(lower=2, upper=2**7 + 1),
- "init_value": 20,
- },
- "learning_rate": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1.0),
- "init_value": 0.1,
- },
- "log_max_bin": { # log transformed with base 2
- "domain": tune.lograndint(lower=3, upper=11),
- "init_value": 8,
- },
- "colsample_bytree": {
- "domain": tune.uniform(lower=0.01, upper=1.0),
- "init_value": 1.0,
- },
- "reg_alpha": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1024),
- "init_value": 1 / 1024,
- },
- "reg_lambda": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1024),
- "init_value": 1.0,
- },
- }
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- if "log_max_bin" in params:
- params["max_bin"] = (1 << params.pop("log_max_bin")) - 1
- return params
-
- @classmethod
- def size(cls, config):
- num_leaves = int(
- round(
- config.get("num_leaves")
- or config.get("max_leaves")
- or 1 << config.get("max_depth", 16)
- )
- )
- n_estimators = int(round(config["n_estimators"]))
- return (num_leaves * 3 + (num_leaves - 1) * 4 + 1.0) * n_estimators * 8
-
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
- if "verbose" not in self.params:
- self.params["verbose"] = -1
- if "regression" == task:
- from lightgbm import LGBMRegressor
-
- self.estimator_class = LGBMRegressor
- elif "rank" == task:
- from lightgbm import LGBMRanker
-
- self.estimator_class = LGBMRanker
- else:
- from lightgbm import LGBMClassifier
-
- self.estimator_class = LGBMClassifier
- self._time_per_iter = None
- self._train_size = 0
- self._mem_per_iter = -1
- self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0, 0) is not None
-
- def _preprocess(self, X):
- if (
- not isinstance(X, DataFrame)
- and issparse(X)
- and np.issubdtype(X.dtype, np.integer)
- ):
- X = X.astype(float)
- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
- # numpy array is not of numeric dtype
- X = DataFrame(X)
- for col in X.columns:
- if isinstance(X[col][0], str):
- X[col] = X[col].astype("category").cat.codes
- X = X.to_numpy()
- return X
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- start_time = time.time()
- deadline = start_time + budget if budget else np.inf
- n_iter = self.params.get(self.ITER_HP, self.DEFAULT_ITER)
- trained = False
- if not self.HAS_CALLBACK:
- mem0 = psutil.virtual_memory().available if psutil is not None else 1
- if (
- (
- not self._time_per_iter
- or abs(self._train_size - X_train.shape[0]) > 4
- )
- and budget is not None
- or self._mem_per_iter < 0
- and psutil is not None
- ) and n_iter > 1:
- self.params[self.ITER_HP] = 1
- self._t1 = self._fit(X_train, y_train, **kwargs)
- if budget is not None and self._t1 >= budget or n_iter == 1:
- return self._t1
- mem1 = psutil.virtual_memory().available if psutil is not None else 1
- self._mem1 = mem0 - mem1
- self.params[self.ITER_HP] = min(n_iter, 4)
- self._t2 = self._fit(X_train, y_train, **kwargs)
- mem2 = psutil.virtual_memory().available if psutil is not None else 1
- self._mem2 = max(mem0 - mem2, self._mem1)
- # if self._mem1 <= 0:
- # self._mem_per_iter = self._mem2 / (self.params[self.ITER_HP] + 1)
- # elif self._mem2 <= 0:
- # self._mem_per_iter = self._mem1
- # else:
- self._mem_per_iter = min(
- self._mem1, self._mem2 / self.params[self.ITER_HP]
- )
- # if self._mem_per_iter <= 1 and psutil is not None:
- # n_iter = self.params[self.ITER_HP]
- self._time_per_iter = (
- (self._t2 - self._t1) / (self.params[self.ITER_HP] - 1)
- if self._t2 > self._t1
- else self._t1
- if self._t1
- else 0.001
- )
- self._train_size = X_train.shape[0]
- if (
- budget is not None
- and self._t1 + self._t2 >= budget
- or n_iter == self.params[self.ITER_HP]
- ):
- # self.params[self.ITER_HP] = n_iter
- return time.time() - start_time
- trained = True
- # logger.debug(mem0)
- # logger.debug(self._mem_per_iter)
- if n_iter > 1:
- max_iter = min(
- n_iter,
- int(
- (budget - time.time() + start_time - self._t1)
- / self._time_per_iter
- + 1
- )
- if budget is not None
- else n_iter,
- int((1 - free_mem_ratio) * mem0 / self._mem_per_iter)
- if psutil is not None and self._mem_per_iter > 0
- else n_iter,
- )
- if trained and max_iter <= self.params[self.ITER_HP]:
- return time.time() - start_time
- # when not trained, train at least one iter
- self.params[self.ITER_HP] = max(max_iter, 1)
- if self.HAS_CALLBACK:
- kwargs_callbacks = kwargs.get("callbacks")
- if kwargs_callbacks:
- callbacks = kwargs_callbacks + self._callbacks(
- start_time, deadline, free_mem_ratio
- )
- kwargs.pop("callbacks")
- else:
- callbacks = self._callbacks(start_time, deadline, free_mem_ratio)
- if isinstance(self, XGBoostSklearnEstimator):
- from xgboost import __version__
-
- if __version__ >= "1.6.0":
- # since xgboost>=1.6.0, callbacks can't be passed in fit()
- self.params["callbacks"] = callbacks
- callbacks = None
- self._fit(
- X_train,
- y_train,
- callbacks=callbacks,
- **kwargs,
- )
- if callbacks is None:
- # for xgboost>=1.6.0, pop callbacks to enable pickle
- callbacks = self.params.pop("callbacks")
- self._model.set_params(callbacks=callbacks[:-1])
- best_iteration = (
- self._model.get_booster().best_iteration
- if isinstance(self, XGBoostSklearnEstimator)
- else self._model.best_iteration_
- )
- if best_iteration is not None:
- self._model.set_params(n_estimators=best_iteration + 1)
- else:
- self._fit(X_train, y_train, **kwargs)
- train_time = time.time() - start_time
- return train_time
-
- def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:
- return [partial(self._callback, start_time, deadline, free_mem_ratio)]
-
- def _callback(self, start_time, deadline, free_mem_ratio, env) -> None:
- from lightgbm.callback import EarlyStopException
-
- now = time.time()
- if env.iteration == 0:
- self._time_per_iter = now - start_time
- if now + self._time_per_iter > deadline:
- raise EarlyStopException(env.iteration, env.evaluation_result_list)
- if psutil is not None:
- mem = psutil.virtual_memory()
- if mem.available / mem.total < free_mem_ratio:
- raise EarlyStopException(env.iteration, env.evaluation_result_list)
-
-
-class XGBoostEstimator(SKLearnEstimator):
- """The class for tuning XGBoost regressor, not using sklearn API."""
-
- DEFAULT_ITER = 10
-
- @classmethod
- def search_space(cls, data_size, **params):
- upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower
- return {
- "n_estimators": {
- "domain": tune.lograndint(lower=4, upper=upper),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- "max_leaves": {
- "domain": tune.lograndint(lower=4, upper=upper),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- "max_depth": {
- "domain": tune.choice([0, 6, 12]),
- "init_value": 0,
- },
- "min_child_weight": {
- "domain": tune.loguniform(lower=0.001, upper=128),
- "init_value": 1.0,
- },
- "learning_rate": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1.0),
- "init_value": 0.1,
- },
- "subsample": {
- "domain": tune.uniform(lower=0.1, upper=1.0),
- "init_value": 1.0,
- },
- "colsample_bylevel": {
- "domain": tune.uniform(lower=0.01, upper=1.0),
- "init_value": 1.0,
- },
- "colsample_bytree": {
- "domain": tune.uniform(lower=0.01, upper=1.0),
- "init_value": 1.0,
- },
- "reg_alpha": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1024),
- "init_value": 1 / 1024,
- },
- "reg_lambda": {
- "domain": tune.loguniform(lower=1 / 1024, upper=1024),
- "init_value": 1.0,
- },
- }
-
- @classmethod
- def size(cls, config):
- return LGBMEstimator.size(config)
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 1.6
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- max_depth = params["max_depth"] = params.get("max_depth", 0)
- if max_depth == 0:
- params["grow_policy"] = params.get("grow_policy", "lossguide")
- params["tree_method"] = params.get("tree_method", "hist")
- # params["booster"] = params.get("booster", "gbtree")
- params["use_label_encoder"] = params.get("use_label_encoder", False)
- if "n_jobs" in config:
- params["nthread"] = params.pop("n_jobs")
- return params
-
- def __init__(
- self,
- task="regression",
- **config,
- ):
- super().__init__(task, **config)
- self.params["verbosity"] = 0
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import xgboost as xgb
-
- start_time = time.time()
- deadline = start_time + budget if budget else np.inf
- if issparse(X_train):
- if xgb.__version__ < "1.6.0":
- # "auto" fails for sparse input since xgboost 1.6.0
- self.params["tree_method"] = "auto"
- else:
- X_train = self._preprocess(X_train)
- if "sample_weight" in kwargs:
- dtrain = xgb.DMatrix(X_train, label=y_train, weight=kwargs["sample_weight"])
- else:
- dtrain = xgb.DMatrix(X_train, label=y_train)
-
- objective = self.params.get("objective")
- if isinstance(objective, str):
- obj = None
- else:
- obj = objective
- if "objective" in self.params:
- del self.params["objective"]
- _n_estimators = self.params.pop("n_estimators")
- callbacks = XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)
- if callbacks:
- self._model = xgb.train(
- self.params,
- dtrain,
- _n_estimators,
- obj=obj,
- callbacks=callbacks,
- )
- self.params["n_estimators"] = self._model.best_iteration + 1
- else:
- self._model = xgb.train(self.params, dtrain, _n_estimators, obj=obj)
- self.params["n_estimators"] = _n_estimators
- self.params["objective"] = objective
- del dtrain
- train_time = time.time() - start_time
- return train_time
-
- def predict(self, X, **kwargs):
- import xgboost as xgb
-
- if not issparse(X):
- X = self._preprocess(X)
- dtest = xgb.DMatrix(X)
- return super().predict(dtest, **kwargs)
-
- @classmethod
- def _callbacks(cls, start_time, deadline, free_mem_ratio):
- try:
- from xgboost.callback import TrainingCallback
- except ImportError: # for xgboost<1.3
- return None
-
- class ResourceLimit(TrainingCallback):
- def after_iteration(self, model, epoch, evals_log) -> bool:
- now = time.time()
- if epoch == 0:
- self._time_per_iter = now - start_time
- if now + self._time_per_iter > deadline:
- return True
- if psutil is not None:
- mem = psutil.virtual_memory()
- if mem.available / mem.total < free_mem_ratio:
- return True
- return False
-
- return [ResourceLimit()]
-
-
-class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):
- """The class for tuning XGBoost with unlimited depth, using sklearn API."""
-
- DEFAULT_ITER = 10
-
- @classmethod
- def search_space(cls, data_size, **params):
- space = XGBoostEstimator.search_space(data_size)
- space.pop("max_depth")
- return space
-
- @classmethod
- def cost_relative2lgbm(cls):
- return XGBoostEstimator.cost_relative2lgbm()
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- max_depth = params["max_depth"] = params.get("max_depth", 0)
- if max_depth == 0:
- params["grow_policy"] = params.get("grow_policy", "lossguide")
- params["tree_method"] = params.get("tree_method", "hist")
- params["use_label_encoder"] = params.get("use_label_encoder", False)
- return params
-
- def __init__(
- self,
- task="binary",
- **config,
- ):
- super().__init__(task, **config)
- del self.params["verbose"]
- self.params["verbosity"] = 0
- import xgboost as xgb
-
- self.estimator_class = xgb.XGBRegressor
- if "rank" == task:
- self.estimator_class = xgb.XGBRanker
- elif task in CLASSIFICATION:
- self.estimator_class = xgb.XGBClassifier
- self._xgb_version = xgb.__version__
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- if issparse(X_train) and self._xgb_version < "1.6.0":
- # "auto" fails for sparse input since xgboost 1.6.0
- self.params["tree_method"] = "auto"
- if kwargs.get("gpu_per_trial"):
- self.params["tree_method"] = "gpu_hist"
- kwargs.pop("gpu_per_trial")
- return super().fit(X_train, y_train, budget, free_mem_ratio, **kwargs)
-
- def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:
- return XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)
-
-
-class XGBoostLimitDepthEstimator(XGBoostSklearnEstimator):
- """The class for tuning XGBoost with limited depth, using sklearn API."""
-
- @classmethod
- def search_space(cls, data_size, **params):
- space = XGBoostEstimator.search_space(data_size)
- space.pop("max_leaves")
- upper = max(6, int(np.log2(data_size[0])))
- space["max_depth"] = {
- "domain": tune.randint(lower=1, upper=min(upper, 16)),
- "init_value": 6,
- "low_cost_init_value": 1,
- }
- space["learning_rate"]["init_value"] = 0.3
- space["n_estimators"]["init_value"] = 10
- return space
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 64
-
-
-class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
- """The class for tuning Random Forest."""
-
- HAS_CALLBACK = False
- nrows = 101
-
- @classmethod
- def search_space(cls, data_size, task, **params):
- RandomForestEstimator.nrows = int(data_size[0])
- upper = min(2048, RandomForestEstimator.nrows)
- init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1
- lower = min(0.1, init)
- space = {
- "n_estimators": {
- "domain": tune.lograndint(lower=4, upper=max(5, upper)),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- "max_features": {
- "domain": tune.loguniform(lower=lower, upper=1.0),
- "init_value": init,
- },
- "max_leaves": {
- "domain": tune.lograndint(
- lower=4,
- upper=max(5, min(32768, RandomForestEstimator.nrows >> 1)), #
- ),
- "init_value": 4,
- "low_cost_init_value": 4,
- },
- }
- if task in CLASSIFICATION:
- space["criterion"] = {
- "domain": tune.choice(["gini", "entropy"]),
- # "init_value": "gini",
- }
- return space
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 2
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- if "max_leaves" in params:
- params["max_leaf_nodes"] = params.get(
- "max_leaf_nodes", params.pop("max_leaves")
- )
- if self._task not in CLASSIFICATION and "criterion" in config:
- params.pop("criterion")
- if "random_state" not in params:
- params["random_state"] = 12032022
- return params
-
- def __init__(
- self,
- task="binary",
- **params,
- ):
- super().__init__(task, **params)
- self.params["verbose"] = 0
- self.estimator_class = RandomForestRegressor
- if task in CLASSIFICATION:
- self.estimator_class = RandomForestClassifier
-
-
-class ExtraTreesEstimator(RandomForestEstimator):
- """The class for tuning Extra Trees."""
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 1.9
-
- def __init__(self, task="binary", **params):
- super().__init__(task, **params)
- if "regression" in task:
- self.estimator_class = ExtraTreesRegressor
- else:
- self.estimator_class = ExtraTreesClassifier
-
-
-class LRL1Classifier(SKLearnEstimator):
- """The class for tuning Logistic Regression with L1 regularization."""
-
- @classmethod
- def search_space(cls, **params):
- return {
- "C": {
- "domain": tune.loguniform(lower=0.03125, upper=32768.0),
- "init_value": 1.0,
- },
- }
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 160
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- params["tol"] = params.get("tol", 0.0001)
- params["solver"] = params.get("solver", "saga")
- params["penalty"] = params.get("penalty", "l1")
- return params
-
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
- assert task in CLASSIFICATION, "LogisticRegression for classification task only"
- self.estimator_class = LogisticRegression
-
-
-class LRL2Classifier(SKLearnEstimator):
- """The class for tuning Logistic Regression with L2 regularization."""
-
- limit_resource = True
-
- @classmethod
- def search_space(cls, **params):
- return LRL1Classifier.search_space(**params)
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 25
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- params["tol"] = params.get("tol", 0.0001)
- params["solver"] = params.get("solver", "lbfgs")
- params["penalty"] = params.get("penalty", "l2")
- return params
-
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
- assert task in CLASSIFICATION, "LogisticRegression for classification task only"
- self.estimator_class = LogisticRegression
-
-
-class CatBoostEstimator(BaseEstimator):
- """The class for tuning CatBoost."""
-
- ITER_HP = "n_estimators"
- DEFAULT_ITER = 1000
-
- @classmethod
- def search_space(cls, data_size, **params):
- upper = max(min(round(1500000 / data_size[0]), 150), 12)
- return {
- "early_stopping_rounds": {
- "domain": tune.lograndint(lower=10, upper=upper),
- "init_value": 10,
- "low_cost_init_value": 10,
- },
- "learning_rate": {
- "domain": tune.loguniform(lower=0.005, upper=0.2),
- "init_value": 0.1,
- },
- "n_estimators": {
- "domain": 8192,
- "init_value": 8192,
- },
- }
-
- @classmethod
- def size(cls, config):
- n_estimators = config.get("n_estimators", 8192)
- max_leaves = 64
- return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 15
-
- def _preprocess(self, X):
- if isinstance(X, DataFrame):
- cat_columns = X.select_dtypes(include=["category"]).columns
- if not cat_columns.empty:
- X = X.copy()
- X[cat_columns] = X[cat_columns].apply(
- lambda x: x.cat.rename_categories(
- [
- str(c) if isinstance(c, float) else c
- for c in x.cat.categories
- ]
- )
- )
- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
- # numpy array is not of numeric dtype
- X = DataFrame(X)
- for col in X.columns:
- if isinstance(X[col][0], str):
- X[col] = X[col].astype("category").cat.codes
- X = X.to_numpy()
- return X
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- params["n_estimators"] = params.get("n_estimators", 8192)
- if "n_jobs" in params:
- params["thread_count"] = params.pop("n_jobs")
- return params
-
- def __init__(
- self,
- task="binary",
- **config,
- ):
- super().__init__(task, **config)
- self.params.update(
- {
- "verbose": config.get("verbose", False),
- "random_seed": config.get("random_seed", 10242048),
- }
- )
- from catboost import CatBoostRegressor
-
- self.estimator_class = CatBoostRegressor
- if task in CLASSIFICATION:
- from catboost import CatBoostClassifier
-
- self.estimator_class = CatBoostClassifier
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- start_time = time.time()
- deadline = start_time + budget if budget else np.inf
- train_dir = f"catboost_{str(start_time)}"
- X_train = self._preprocess(X_train)
- if isinstance(X_train, DataFrame):
- cat_features = list(X_train.select_dtypes(include="category").columns)
- else:
- cat_features = []
- use_best_model = kwargs.get("use_best_model", True)
- n = (
- max(int(len(y_train) * 0.9), len(y_train) - 1000)
- if use_best_model
- else len(y_train)
- )
- X_tr, y_tr = X_train[:n], y_train[:n]
- from catboost import Pool, __version__
-
- eval_set = (
- Pool(data=X_train[n:], label=y_train[n:], cat_features=cat_features)
- if use_best_model
- else None
- )
- if "sample_weight" in kwargs:
- weight = kwargs["sample_weight"]
- if weight is not None:
- kwargs["sample_weight"] = weight[:n]
- else:
- weight = None
-
- model = self.estimator_class(train_dir=train_dir, **self.params)
- if __version__ >= "0.26":
- model.fit(
- X_tr,
- y_tr,
- cat_features=cat_features,
- eval_set=eval_set,
- callbacks=CatBoostEstimator._callbacks(
- start_time, deadline, free_mem_ratio if use_best_model else None
- ),
- **kwargs,
- )
- else:
- model.fit(
- X_tr,
- y_tr,
- cat_features=cat_features,
- eval_set=eval_set,
- **kwargs,
- )
- shutil.rmtree(train_dir, ignore_errors=True)
- if weight is not None:
- kwargs["sample_weight"] = weight
- self._model = model
- self.params[self.ITER_HP] = self._model.tree_count_
- train_time = time.time() - start_time
- return train_time
-
- @classmethod
- def _callbacks(cls, start_time, deadline, free_mem_ratio):
- class ResourceLimit:
- def after_iteration(self, info) -> bool:
- now = time.time()
- if info.iteration == 1:
- self._time_per_iter = now - start_time
- if now + self._time_per_iter > deadline:
- return False
- if psutil is not None and free_mem_ratio is not None:
- mem = psutil.virtual_memory()
- if mem.available / mem.total < free_mem_ratio:
- return False
- return True # can continue
-
- return [ResourceLimit()]
-
-
-class KNeighborsEstimator(BaseEstimator):
- @classmethod
- def search_space(cls, data_size, **params):
- upper = min(512, int(data_size[0] / 2))
- return {
- "n_neighbors": {
- "domain": tune.lograndint(lower=1, upper=max(2, upper)),
- "init_value": 5,
- "low_cost_init_value": 1,
- },
- }
-
- @classmethod
- def cost_relative2lgbm(cls):
- return 30
-
- def config2params(self, config: dict) -> dict:
- params = super().config2params(config)
- params["weights"] = params.get("weights", "distance")
- return params
-
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
- if task in CLASSIFICATION:
- from sklearn.neighbors import KNeighborsClassifier
-
- self.estimator_class = KNeighborsClassifier
- else:
- from sklearn.neighbors import KNeighborsRegressor
-
- self.estimator_class = KNeighborsRegressor
-
- def _preprocess(self, X):
- if isinstance(X, DataFrame):
- cat_columns = X.select_dtypes(["category"]).columns
- if X.shape[1] == len(cat_columns):
- raise ValueError("kneighbor requires at least one numeric feature")
- X = X.drop(cat_columns, axis=1)
- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":
- # drop categocial columns if any
- X = DataFrame(X)
- cat_columns = []
- for col in X.columns:
- if isinstance(X[col][0], str):
- cat_columns.append(col)
- X = X.drop(cat_columns, axis=1)
- X = X.to_numpy()
- return X
-
-
-class Prophet(SKLearnEstimator):
- """The class for tuning Prophet."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "changepoint_prior_scale": {
- "domain": tune.loguniform(lower=0.001, upper=0.05),
- "init_value": 0.05,
- "low_cost_init_value": 0.001,
- },
- "seasonality_prior_scale": {
- "domain": tune.loguniform(lower=0.01, upper=10),
- "init_value": 10,
- },
- "holidays_prior_scale": {
- "domain": tune.loguniform(lower=0.01, upper=10),
- "init_value": 10,
- },
- "seasonality_mode": {
- "domain": tune.choice(["additive", "multiplicative"]),
- "init_value": "multiplicative",
- },
- }
- return space
-
- def __init__(self, task="ts_forecast", n_jobs=1, **params):
- super().__init__(task, **params)
-
- def _join(self, X_train, y_train):
- assert TS_TIMESTAMP_COL in X_train, (
- "Dataframe for training ts_forecast model must have column"
- f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'
- )
- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
- train_df = X_train.join(y_train)
- return train_df
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- from prophet import Prophet
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- cols = list(train_df)
- cols.remove(TS_TIMESTAMP_COL)
- cols.remove(TS_VALUE_COL)
- logging.getLogger("prophet").setLevel(logging.WARNING)
- model = Prophet(**self.params)
- for regressor in cols:
- model.add_regressor(regressor)
- with suppress_stdout_stderr():
- model.fit(train_df)
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def predict(self, X, **kwargs):
- if isinstance(X, int):
- raise ValueError(
- "predict() with steps is only supported for arima/sarimax."
- " For Prophet, pass a dataframe with the first column containing"
- " the timestamp values."
- )
- if self._model is not None:
- X = self._preprocess(X)
- forecast = self._model.predict(X, **kwargs)
- return forecast["yhat"]
- else:
- logger.warning(
- "Estimator is not fit yet. Please run fit() before predict()."
- )
- return np.ones(X.shape[0])
-
- def score(self, X_val: DataFrame, y_val: Series, **kwargs):
- from sklearn.metrics import r2_score
- from .ml import metric_loss_score
-
- y_pred = self.predict(X_val, **kwargs)
- self._metric = kwargs.get("metric", None)
- if self._metric:
- return metric_loss_score(self._metric, y_pred, y_val)
- else:
- return r2_score(y_pred, y_val)
-
-
-class ARIMA(Prophet):
- """The class for tuning ARIMA."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "p": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "d": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- }
- return space
-
- def _join(self, X_train, y_train):
- train_df = super()._join(X_train, y_train)
- train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])
- train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)
- return train_df
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
-
- warnings.filterwarnings("ignore")
- from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- regressors = list(train_df)
- regressors.remove(TS_VALUE_COL)
- if regressors:
- model = ARIMA_estimator(
- train_df[[TS_VALUE_COL]],
- exog=train_df[regressors],
- order=(self.params["p"], self.params["d"], self.params["q"]),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- else:
- model = ARIMA_estimator(
- train_df,
- order=(self.params["p"], self.params["d"], self.params["q"]),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- with suppress_stdout_stderr():
- model = model.fit()
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def predict(self, X, **kwargs):
- if self._model is not None:
- if isinstance(X, int):
- forecast = self._model.forecast(steps=X)
- elif isinstance(X, DataFrame):
- start = X[TS_TIMESTAMP_COL].iloc[0]
- end = X[TS_TIMESTAMP_COL].iloc[-1]
- if len(X.columns) > 1:
- X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))
- regressors = list(X)
- forecast = self._model.predict(
- start=start, end=end, exog=X[regressors], **kwargs
- )
- else:
- forecast = self._model.predict(start=start, end=end, **kwargs)
- else:
- raise ValueError(
- "X needs to be either a pandas Dataframe with dates as the first column"
- " or an int number of periods for predict()."
- )
- return forecast
- else:
- return np.ones(X if isinstance(X, int) else X.shape[0])
-
-
-class SARIMAX(ARIMA):
- """The class for tuning SARIMA."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "p": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "d": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "P": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "D": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "Q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "s": {
- "domain": tune.choice([1, 4, 6, 12]),
- "init_value": 12,
- },
- }
- return space
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
-
- warnings.filterwarnings("ignore")
- from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- regressors = list(train_df)
- regressors.remove(TS_VALUE_COL)
- if regressors:
- model = SARIMAX_estimator(
- train_df[[TS_VALUE_COL]],
- exog=train_df[regressors],
- order=(self.params["p"], self.params["d"], self.params["q"]),
- seasonal_order=(
- self.params["P"],
- self.params["D"],
- self.params["Q"],
- self.params["s"],
- ),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- else:
- model = SARIMAX_estimator(
- train_df,
- order=(self.params["p"], self.params["d"], self.params["q"]),
- seasonal_order=(
- self.params["P"],
- self.params["D"],
- self.params["Q"],
- self.params["s"],
- ),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- with suppress_stdout_stderr():
- model = model.fit()
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
-
-class TS_SKLearn(SKLearnEstimator):
- """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""
-
- base_class = SKLearnEstimator
-
- @classmethod
- def search_space(cls, data_size, pred_horizon, **params):
- space = cls.base_class.search_space(data_size, **params)
- space.update(
- {
- "optimize_for_horizon": {
- "domain": tune.choice([True, False]),
- "init_value": False,
- "low_cost_init_value": False,
- },
- "lags": {
- "domain": tune.randint(
- lower=1, upper=max(2, int(np.sqrt(data_size[0])))
- ),
- "init_value": 3,
- },
- }
- )
- return space
-
- def __init__(self, task="ts_forecast", **params):
- super().__init__(task, **params)
- self.hcrystaball_model = None
- self.ts_task = (
- "regression" if task in TS_FORECASTREGRESSION else "classification"
- )
-
- def transform_X(self, X):
- cols = list(X)
- if len(cols) == 1:
- ds_col = cols[0]
- X = DataFrame(index=X[ds_col])
- elif len(cols) > 1:
- ds_col = cols[0]
- exog_cols = cols[1:]
- X = X[exog_cols].set_index(X[ds_col])
- return X
-
- def _fit(self, X_train, y_train, budget=None, **kwargs):
- from hcrystalball.wrappers import get_sklearn_wrapper
-
- X_train = self.transform_X(X_train)
- X_train = self._preprocess(X_train)
- params = self.params.copy()
- lags = params.pop("lags")
- optimize_for_horizon = params.pop("optimize_for_horizon")
- estimator = self.base_class(task=self.ts_task, **params)
- self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)
- self.hcrystaball_model.lags = int(lags)
- self.hcrystaball_model.fit(X_train, y_train)
- if optimize_for_horizon:
- # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon
- model_list = []
- for i in range(1, kwargs["period"] + 1):
- (
- X_fit,
- y_fit,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
- X_train, y_train, i
- )
- self.hcrystaball_model.model.set_params(**estimator.params)
- model = self.hcrystaball_model.model.fit(X_fit, y_fit)
- model_list.append(model)
- self._model = model_list
- else:
- (
- X_fit,
- y_fit,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
- X_train, y_train, kwargs["period"]
- )
- self.hcrystaball_model.model.set_params(**estimator.params)
- model = self.hcrystaball_model.model.fit(X_fit, y_fit)
- self._model = model
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- current_time = time.time()
- self._fit(X_train, y_train, budget=budget, **kwargs)
- train_time = time.time() - current_time
- return train_time
-
- def predict(self, X, **kwargs):
- if self._model is not None:
- X = self.transform_X(X)
- X = self._preprocess(X)
- if isinstance(self._model, list):
- assert len(self._model) == len(
- X
- ), "Model is optimized for horizon, length of X must be equal to `period`."
- preds = []
- for i in range(1, len(self._model) + 1):
- (
- X_pred,
- _,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(
- X.iloc[:i, :]
- )
- preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])
- forecast = DataFrame(
- data=np.asarray(preds).reshape(-1, 1),
- columns=[self.hcrystaball_model.name],
- index=X.index,
- )
- else:
- (
- X_pred,
- _,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)
- forecast = self._model.predict(X_pred, **kwargs)
- return forecast
- else:
- logger.warning(
- "Estimator is not fit yet. Please run fit() before predict()."
- )
- return np.ones(X.shape[0])
-
-
-class LGBM_TS(TS_SKLearn):
- """The class for tuning LGBM Regressor for time-series forecasting"""
-
- base_class = LGBMEstimator
-
-
-class XGBoost_TS(TS_SKLearn):
- """The class for tuning XGBoost Regressor for time-series forecasting"""
-
- base_class = XGBoostSklearnEstimator
-
-
-# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball
-# class CatBoost_TS_Regressor(TS_Regressor):
-# base_class = CatBoostEstimator
-
-
-class RF_TS(TS_SKLearn):
- """The class for tuning Random Forest Regressor for time-series forecasting"""
-
- base_class = RandomForestEstimator
-
-
-class ExtraTrees_TS(TS_SKLearn):
- """The class for tuning Extra Trees Regressor for time-series forecasting"""
-
- base_class = ExtraTreesEstimator
-
-
-class XGBoostLimitDepth_TS(TS_SKLearn):
- """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""
-
- base_class = XGBoostLimitDepthEstimator
-
-
-class TemporalFusionTransformerEstimator(SKLearnEstimator):
- """The class for tuning Temporal Fusion Transformer"""
-
- @classmethod
- def search_space(cls, data_size, pred_horizon, **params):
- space = {
- "gradient_clip_val": {
- "domain": tune.loguniform(lower=0.01, upper=100.0),
- "init_value": 0.01,
- },
- "hidden_size": {
- "domain": tune.lograndint(lower=8, upper=512),
- "init_value": 16,
- },
- "hidden_continuous_size": {
- "domain": tune.randint(lower=1, upper=65),
- "init_value": 8,
- },
- "attention_head_size": {
- "domain": tune.randint(lower=1, upper=5),
- "init_value": 4,
- },
- "dropout": {
- "domain": tune.uniform(lower=0.1, upper=0.3),
- "init_value": 0.1,
- },
- "learning_rate": {
- "domain": tune.loguniform(lower=0.00001, upper=1.0),
- "init_value": 0.001,
- },
- }
- return space
-
- def transform_ds(self, X_train, y_train, **kwargs):
- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
- self.data = X_train.join(y_train)
-
- max_prediction_length = kwargs["period"]
- self.max_encoder_length = kwargs["max_encoder_length"]
- training_cutoff = self.data["time_idx"].max() - max_prediction_length
-
- from pytorch_forecasting import TimeSeriesDataSet
- from pytorch_forecasting.data import GroupNormalizer
-
- self.group_ids = kwargs["group_ids"].copy()
- training = TimeSeriesDataSet(
- self.data[lambda x: x.time_idx <= training_cutoff],
- time_idx="time_idx",
- target=TS_VALUE_COL,
- group_ids=self.group_ids,
- min_encoder_length=kwargs.get(
- "min_encoder_length", self.max_encoder_length // 2
- ), # keep encoder length long (as it is in the validation set)
- max_encoder_length=self.max_encoder_length,
- min_prediction_length=1,
- max_prediction_length=max_prediction_length,
- static_categoricals=kwargs.get("static_categoricals", []),
- static_reals=kwargs.get("static_reals", []),
- time_varying_known_categoricals=kwargs.get(
- "time_varying_known_categoricals", []
- ),
- time_varying_known_reals=kwargs.get("time_varying_known_reals", []),
- time_varying_unknown_categoricals=kwargs.get(
- "time_varying_unknown_categoricals", []
- ),
- time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),
- variable_groups=kwargs.get(
- "variable_groups", {}
- ), # group of categorical variables can be treated as one variable
- lags=kwargs.get("lags", {}),
- target_normalizer=GroupNormalizer(
- groups=kwargs["group_ids"], transformation="softplus"
- ), # use softplus and normalize by group
- add_relative_time_idx=True,
- add_target_scales=True,
- add_encoder_length=True,
- )
-
- # create validation set (predict=True) which means to predict the last max_prediction_length points in time
- # for each series
- validation = TimeSeriesDataSet.from_dataset(
- training, self.data, predict=True, stop_randomization=True
- )
-
- # create dataloaders for model
- batch_size = kwargs.get("batch_size", 64)
- train_dataloader = training.to_dataloader(
- train=True, batch_size=batch_size, num_workers=0
- )
- val_dataloader = validation.to_dataloader(
- train=False, batch_size=batch_size * 10, num_workers=0
- )
-
- return training, train_dataloader, val_dataloader
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
- import pytorch_lightning as pl
- from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
- import torch
- from pytorch_forecasting import TemporalFusionTransformer
- from pytorch_forecasting.metrics import QuantileLoss
-
- warnings.filterwarnings("ignore")
- current_time = time.time()
- training, train_dataloader, val_dataloader = self.transform_ds(
- X_train, y_train, **kwargs
- )
- params = self.params.copy()
- gradient_clip_val = params.pop("gradient_clip_val")
- params.pop("n_jobs")
- max_epochs = kwargs.get("max_epochs", 20)
- early_stop_callback = EarlyStopping(
- monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min"
- )
-
- def _fit(log):
- default_trainer_kwargs = dict(
- gpus=kwargs.get("gpu_per_trial", [0])
- if torch.cuda.is_available()
- else None,
- max_epochs=max_epochs,
- gradient_clip_val=gradient_clip_val,
- callbacks=[LearningRateMonitor(), early_stop_callback]
- if log
- else [early_stop_callback],
- logger=log,
- )
- trainer = pl.Trainer(
- **default_trainer_kwargs,
- )
- tft = TemporalFusionTransformer.from_dataset(
- training,
- **params,
- lstm_layers=2, # 2 is mostly optimal according to documentation
- output_size=7, # 7 quantiles by default
- loss=QuantileLoss(),
- log_interval=10 if log else 0,
- # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches
- reduce_on_plateau_patience=4,
- )
- # fit network
- trainer.fit(
- tft,
- train_dataloaders=train_dataloader,
- val_dataloaders=val_dataloader,
- )
- return trainer
-
- # try:
- # from pytorch_lightning.loggers import TensorBoardLogger
-
- # logger = TensorBoardLogger(
- # kwargs.get("log_dir", "lightning_logs")
- # ) # logging results to a tensorboard
- # trainer = _fit(log=logger)
- # except ValueError:
- # issue with pytorch forecasting model log_prediction() function
- # pytorch-forecasting issue #1145
- trainer = _fit(log=False)
- best_model_path = trainer.checkpoint_callback.best_model_path
- best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
- train_time = time.time() - current_time
- self._model = best_tft
- return train_time
-
- def predict(self, X):
- import pandas as pd
-
- ids = self.group_ids.copy()
- ids.append(TS_TIMESTAMP_COL)
- encoder_data = self.data[
- lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length
- ]
- # following pytorchforecasting example, make all target values equal to the last data
- last_data_cols = self.group_ids.copy()
- last_data_cols.append(TS_VALUE_COL)
- last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]
- decoder_data = X
- if "time_idx" not in decoder_data:
- decoder_data = add_time_idx_col(decoder_data)
- decoder_data["time_idx"] += (
- encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()
- )
- # decoder_data[TS_VALUE_COL] = 0
- decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)
- decoder_data = decoder_data.sort_values(ids)
- new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)
- new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")
- new_raw_predictions = self._model.predict(new_prediction_data)
- index = [decoder_data[idx].to_numpy() for idx in ids]
- predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)
- return predictions
-
-
-class suppress_stdout_stderr(object):
- def __init__(self):
- # Open a pair of null files
- self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]
- # Save the actual stdout (1) and stderr (2) file descriptors.
- self.save_fds = (os.dup(1), os.dup(2))
-
- def __enter__(self):
- # Assign the null pointers to stdout and stderr.
- os.dup2(self.null_fds[0], 1)
- os.dup2(self.null_fds[1], 2)
-
- def __exit__(self, *_):
- # Re-assign the real stdout/stderr back to (1) and (2)
- os.dup2(self.save_fds[0], 1)
- os.dup2(self.save_fds[1], 2)
- # Close the null files
- os.close(self.null_fds[0])
- os.close(self.null_fds[1])
+import warnings
+
+from flaml.automl.model import *
+
+
+warnings.warn(
+ "Importing from `flaml.model` is deprecated. Please use `flaml.automl.model`.",
+ DeprecationWarning,
+)
diff --git a/test/automl/test_classification.py b/test/automl/test_classification.py
index 19726029e..adb308166 100644
--- a/test/automl/test_classification.py
+++ b/test/automl/test_classification.py
@@ -6,7 +6,7 @@ from sklearn.model_selection import train_test_split
import pandas as pd
from datetime import datetime
from flaml import AutoML
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
from flaml import tune
diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.py
index 34ffb2ec5..4b9c040e4 100644
--- a/test/automl/test_forecast.py
+++ b/test/automl/test_forecast.py
@@ -58,12 +58,12 @@ def test_forecast_automl(budget=5):
print("Predicted labels", y_pred)
print("True labels", y_test)
""" compute different metric values on testing dataset"""
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
mape = sklearn_metric_loss_score("mape", y_pred, y_test)
print("mape", "=", mape)
assert mape <= 0.005, "the mape of flaml should be less than 0.005"
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
@@ -212,10 +212,10 @@ def test_multivariate_forecast_num(budget=5):
print("Predicted labels", y_pred)
print("True labels", y_test)
""" compute different metric values on testing dataset"""
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
print("mape", "=", sklearn_metric_loss_score("mape", y_pred, y_test))
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
@@ -339,13 +339,13 @@ def test_multivariate_forecast_cat(budget=5):
print("Predicted labels", y_pred)
print("True labels", y_test)
""" compute different metric values on testing dataset"""
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
print("mape", "=", sklearn_metric_loss_score("mape", y_pred, y_test))
print("rmse", "=", sklearn_metric_loss_score("rmse", y_pred, y_test))
print("mse", "=", sklearn_metric_loss_score("mse", y_pred, y_test))
print("mae", "=", sklearn_metric_loss_score("mae", y_pred, y_test))
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
@@ -413,12 +413,12 @@ def test_forecast_classification(budget=5):
""" compute predictions of testing dataset """
y_pred = automl.predict(X_test)
""" compute different metric values on testing dataset"""
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
print(y_test)
print(y_pred)
print("accuracy", "=", 1 - sklearn_metric_loss_score("accuracy", y_pred, y_test))
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
@@ -554,7 +554,7 @@ def test_forecast_panel(budget=5):
""" compute predictions of testing dataset """
y_pred = automl.predict(X_test)
""" compute different metric values on testing dataset"""
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
print(y_test)
print(y_pred)
@@ -575,7 +575,7 @@ def test_forecast_panel(budget=5):
# """compute prediction for a specific time series"""
# a01_sku01_preds = automl.predict(X_test[(X_test["agency"] == "Agency_01") & (X_test["sku"] == "SKU_01")])
# print("Agency01 SKU_01 predictions: ", a01_sku01_preds)
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
diff --git a/test/automl/test_multiclass.py b/test/automl/test_multiclass.py
index a0f18bbcb..f6bb38396 100644
--- a/test/automl/test_multiclass.py
+++ b/test/automl/test_multiclass.py
@@ -3,10 +3,10 @@ import numpy as np
import scipy.sparse
from sklearn.datasets import load_iris, load_wine
from flaml import AutoML
-from flaml.data import CLASSIFICATION, get_output_from_log
-from flaml.model import LGBMEstimator, XGBoostSklearnEstimator, SKLearnEstimator
+from flaml.automl.data import CLASSIFICATION, get_output_from_log
+from flaml.automl.model import LGBMEstimator, XGBoostSklearnEstimator, SKLearnEstimator
from flaml import tune
-from flaml.training_log import training_log_reader
+from flaml.automl.training_log import training_log_reader
class MyRegularizedGreedyForest(SKLearnEstimator):
@@ -280,7 +280,7 @@ class TestMultiClass(unittest.TestCase):
estimator = automl_experiment_macro.model
y_pred = estimator.predict(X_train)
y_pred_proba = estimator.predict_proba(X_train)
- from flaml.ml import norm_confusion_matrix, multi_class_curves
+ from flaml.automl.ml import norm_confusion_matrix, multi_class_curves
print(norm_confusion_matrix(y_train, y_pred))
from sklearn.metrics import roc_curve, precision_recall_curve
diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.py
index 09be23ebb..be86b3b5b 100644
--- a/test/automl/test_notebook_example.py
+++ b/test/automl/test_notebook_example.py
@@ -4,7 +4,7 @@ from requests.exceptions import ChunkedEncodingError, SSLError
def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):
- from flaml.data import load_openml_dataset
+ from flaml.automl.data import load_openml_dataset
import urllib3
performance_check_budget = 600
@@ -77,7 +77,7 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):
print("True labels", y_test)
y_pred_proba = automl.predict_proba(X_test)[:, 1]
""" compute different metric values on testing dataset """
- from flaml.ml import sklearn_metric_loss_score
+ from flaml.automl.ml import sklearn_metric_loss_score
accuracy = 1 - sklearn_metric_loss_score("accuracy", y_pred, y_test)
print("accuracy", "=", accuracy)
@@ -87,7 +87,7 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):
print("log_loss", "=", sklearn_metric_loss_score("log_loss", y_pred_proba, y_test))
if budget is None:
assert accuracy >= 0.669, "the accuracy of flaml should be larger than 0.67"
- from flaml.data import get_output_from_log
+ from flaml.automl.data import get_output_from_log
(
time_history,
@@ -119,7 +119,7 @@ def _test_nobudget():
def test_mlflow():
# subprocess.check_call([sys.executable, "-m", "pip", "install", "mlflow"])
import mlflow
- from flaml.data import load_openml_task
+ from flaml.automl.data import load_openml_task
try:
X_train, X_test, y_train, y_test = load_openml_task(
diff --git a/test/automl/test_regression.py b/test/automl/test_regression.py
index 0aca92f15..02d058c67 100644
--- a/test/automl/test_regression.py
+++ b/test/automl/test_regression.py
@@ -6,8 +6,8 @@ from sklearn.datasets import (
)
from flaml import AutoML
-from flaml.data import get_output_from_log
-from flaml.model import XGBoostEstimator
+from flaml.automl.data import get_output_from_log
+from flaml.automl.model import XGBoostEstimator
def logregobj(preds, dtrain):
diff --git a/test/automl/test_training_log.py b/test/automl/test_training_log.py
index 85cccbe34..ff1b42673 100644
--- a/test/automl/test_training_log.py
+++ b/test/automl/test_training_log.py
@@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
from sklearn.datasets import fetch_california_housing
from flaml import AutoML
-from flaml.training_log import training_log_reader
+from flaml.automl.training_log import training_log_reader
class TestTrainingLog(unittest.TestCase):
diff --git a/test/automl/test_warmstart.py b/test/automl/test_warmstart.py
index c443cf6ce..74cf8c37f 100644
--- a/test/automl/test_warmstart.py
+++ b/test/automl/test_warmstart.py
@@ -2,7 +2,7 @@ import unittest
import numpy as np
from sklearn.datasets import load_iris
from flaml import AutoML
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
from flaml import tune
@@ -124,7 +124,7 @@ class TestWarmStart(unittest.TestCase):
print(automl.best_config_per_estimator)
def test_FLAML_sample_size_in_starting_points(self):
- from flaml.data import load_openml_dataset
+ from flaml.automl.data import load_openml_dataset
from flaml import AutoML
X_train, X_test, y_train, y_test = load_openml_dataset(
diff --git a/test/automl/test_xgboost2d.py b/test/automl/test_xgboost2d.py
index 7c2eba1c2..46150a257 100644
--- a/test/automl/test_xgboost2d.py
+++ b/test/automl/test_xgboost2d.py
@@ -3,7 +3,7 @@ import unittest
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split
from flaml.automl import AutoML
-from flaml.model import XGBoostSklearnEstimator
+from flaml.automl.model import XGBoostSklearnEstimator
from flaml import tune
diff --git a/test/automl/test_xgboost2d_sample_size.py b/test/automl/test_xgboost2d_sample_size.py
index bdc7f8c1e..aabba9139 100644
--- a/test/automl/test_xgboost2d_sample_size.py
+++ b/test/automl/test_xgboost2d_sample_size.py
@@ -3,7 +3,7 @@ import unittest
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split
from flaml.automl import AutoML
-from flaml.model import XGBoostSklearnEstimator
+from flaml.automl.model import XGBoostSklearnEstimator
from flaml import tune
diff --git a/test/default/test_defaults.py b/test/default/test_defaults.py
index 77408b695..22ef0a2d3 100644
--- a/test/default/test_defaults.py
+++ b/test/default/test_defaults.py
@@ -5,12 +5,11 @@ from sklearn.model_selection import train_test_split
import pandas as pd
from flaml import AutoML
from flaml.default import (
- portfolio,
- regret,
preprocess_and_suggest_hyperparams,
suggest_hyperparams,
suggest_learner,
)
+from flaml.default import portfolio, regret
def test_greedy_feedback(path="test/default", strategy="greedy-feedback"):
@@ -147,7 +146,7 @@ def test_suggest_regression():
def test_rf():
- from flaml.default.estimator import RandomForestRegressor, RandomForestClassifier
+ from flaml.default import RandomForestRegressor, RandomForestClassifier
X_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)
rf = RandomForestClassifier()
@@ -165,7 +164,7 @@ def test_rf():
def test_extratrees():
- from flaml.default.estimator import ExtraTreesRegressor, ExtraTreesClassifier
+ from flaml.default import ExtraTreesRegressor, ExtraTreesClassifier
X_train, y_train = load_iris(return_X_y=True, as_frame=True)
classifier = ExtraTreesClassifier()
@@ -183,7 +182,7 @@ def test_extratrees():
def test_lgbm():
- from flaml.default.estimator import LGBMRegressor, LGBMClassifier
+ from flaml.default import LGBMRegressor, LGBMClassifier
X_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)
classifier = LGBMClassifier(n_jobs=1)
@@ -203,7 +202,7 @@ def test_lgbm():
def test_xgboost():
- from flaml.default.estimator import XGBRegressor, XGBClassifier
+ from flaml.default import XGBRegressor, XGBClassifier
X_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)
classifier = XGBClassifier(max_depth=0)
diff --git a/test/default_lgbm.py b/test/default_lgbm.py
index 9085db489..105a3d6ec 100644
--- a/test/default_lgbm.py
+++ b/test/default_lgbm.py
@@ -1,6 +1,6 @@
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml.default import LGBMRegressor
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")
lgbm = LGBMRegressor()
diff --git a/test/default_xgb.py b/test/default_xgb.py
index bda7307e5..14a58deda 100644
--- a/test/default_xgb.py
+++ b/test/default_xgb.py
@@ -1,6 +1,6 @@
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml.default import XGBClassifier
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
xgb = XGBClassifier()
diff --git a/test/load_args.py b/test/load_args.py
index 5c7c35378..9ffcba856 100644
--- a/test/load_args.py
+++ b/test/load_args.py
@@ -1,5 +1,5 @@
def test_load_args_sub():
- from flaml.nlp.utils import TrainingArgumentsForAuto
+ from flaml.automl.nlp.huggingface.training_args import TrainingArgumentsForAuto
TrainingArgumentsForAuto.load_args_from_console()
diff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.py
index c653b74c5..0918e0d85 100644
--- a/test/nlp/test_autohf_custom_metric.py
+++ b/test/nlp/test_autohf_custom_metric.py
@@ -19,7 +19,6 @@ def custom_metric(
groups_train=None,
):
from datasets import Dataset
- from flaml.model import TransformersEstimator
if estimator._trainer is None:
trainer = estimator._init_model_for_predict()
diff --git a/test/object_store.py b/test/object_store.py
index 8539d1cb3..dc00914e6 100644
--- a/test/object_store.py
+++ b/test/object_store.py
@@ -1,5 +1,5 @@
from flaml import tune
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
import lightgbm
from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_california_housing
diff --git a/test/ray/distribute_tune.py b/test/ray/distribute_tune.py
index 991082e8e..e78c624b4 100644
--- a/test/ray/distribute_tune.py
+++ b/test/ray/distribute_tune.py
@@ -5,7 +5,7 @@ from sklearn.datasets import load_breast_cancer
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from flaml import tune
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
def train_breast_cancer(config):
diff --git a/test/rep.py b/test/rep.py
index e78980e78..266ac0059 100644
--- a/test/rep.py
+++ b/test/rep.py
@@ -1,5 +1,5 @@
-from flaml.data import load_openml_dataset
-from flaml.ml import ExtraTreesEstimator
+from flaml.automl.data import load_openml_dataset
+from flaml.automl.ml import ExtraTreesEstimator
from flaml import AutoML
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
diff --git a/test/test_model.py b/test/test_model.py
index 9a12b1b2e..b05f5d58a 100644
--- a/test/test_model.py
+++ b/test/test_model.py
@@ -2,7 +2,7 @@ from sklearn.datasets import make_classification
import numpy as np
from pandas import DataFrame
from datetime import datetime
-from flaml.model import (
+from flaml.automl.model import (
KNeighborsEstimator,
LRL2Classifier,
BaseEstimator,
diff --git a/test/tune_example.py b/test/tune_example.py
index a71300692..0dc770790 100644
--- a/test/tune_example.py
+++ b/test/tune_example.py
@@ -1,5 +1,5 @@
from flaml import tune
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
import lightgbm
from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_california_housing
diff --git a/website/docs/Examples/AutoML-for-LightGBM.md b/website/docs/Examples/AutoML-for-LightGBM.md
index ad5c49ab9..0094dd561 100644
--- a/website/docs/Examples/AutoML-for-LightGBM.md
+++ b/website/docs/Examples/AutoML-for-LightGBM.md
@@ -13,7 +13,7 @@ This option is not necessary in general.
```python
from flaml import AutoML
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
# Download [houses dataset](https://www.openml.org/d/537) from OpenML. The task is to predict median price of the house in the region based on demographic composition and a state of housing market in the region.
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir='./')
@@ -25,7 +25,7 @@ settings = {
"estimator_list": ['lgbm'], # list of ML learners; we tune lightgbm in this example
"task": 'regression', # task type
"log_file_name": 'houses_experiment.log', # flaml log file
- "seed": 7654321, # random seed
+ "seed": 7654321, # random seed
}
automl.fit(X_train=X_train, y_train=y_train, **settings)
```
@@ -114,7 +114,8 @@ print('Predicted labels', y_pred)
#### Compute different metric values on testing dataset
```python
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
+
print('r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))
print('mse', '=', sklearn_metric_loss_score('mse', y_pred, y_test))
print('mae', '=', sklearn_metric_loss_score('mae', y_pred, y_test))
@@ -131,7 +132,8 @@ from lightgbm import LGBMRegressor
lgbm = LGBMRegressor()
lgbm.fit(X_train, y_train)
y_pred = lgbm.predict(X_test)
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
+
print('default lgbm r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))
# default lgbm r2 = 0.8296179648694404
```
@@ -141,10 +143,10 @@ print('default lgbm r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test
How does the model accuracy improve as we search for different hyperparameter configurations?
```python
-from flaml.data import get_output_from_log
+from flaml.automl.data import get_output_from_log
import numpy as np
-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \
+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =
get_output_from_log(filename=settings['log_file_name'], time_budget=60)
plt.title('Learning Curve')
plt.xlabel('Wall Clock Time (s)')
@@ -163,11 +165,12 @@ The native API of LightGBM allows one to specify a custom objective function in
```python
import numpy as np
+
# define your customized objective function
def my_loss_obj(y_true, y_pred):
c = 0.5
residual = y_pred - y_true
- grad = c * residual /(np.abs(residual) + c)
+ grad = c * residual / (np.abs(residual) + c)
hess = c ** 2 / (np.abs(residual) + c) ** 2
# rmse grad and hess
grad_rmse = residual
@@ -180,10 +183,12 @@ def my_loss_obj(y_true, y_pred):
hess_mae = 1.0
coef = [0.4, 0.3, 0.3]
- return coef[0] * grad + coef[1] * grad_rmse + coef[2] * grad_mae, \
- coef[0] * hess + coef[1] * hess_rmse + coef[2] * hess_mae
+ return coef[0] * grad + coef[1] * grad_rmse + coef[2] * grad_mae,
+ coef[0] * hess + coef[1] * hess_rmse + coef[2] * hess_mae
+
+
+from flaml.automl.model import LGBMEstimator
-from flaml.model import LGBMEstimator
class MyLGBM(LGBMEstimator):
"""LGBMEstimator with my_loss_obj as the objective function"""
diff --git a/website/docs/Examples/AutoML-for-XGBoost.md b/website/docs/Examples/AutoML-for-XGBoost.md
index 2b7bf6092..774050a53 100644
--- a/website/docs/Examples/AutoML-for-XGBoost.md
+++ b/website/docs/Examples/AutoML-for-XGBoost.md
@@ -13,7 +13,7 @@ This option is not necessary in general.
```python
from flaml import AutoML
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
# Download [houses dataset](https://www.openml.org/d/537) from OpenML. The task is to predict median price of the house in the region based on demographic composition and a state of housing market in the region.
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir='./')
@@ -25,7 +25,7 @@ settings = {
"estimator_list": ['xgboost'], # list of ML learners; we tune XGBoost in this example
"task": 'regression', # task type
"log_file_name": 'houses_experiment.log', # flaml log file
- "seed": 7654321, # random seed
+ "seed": 7654321, # random seed
}
automl.fit(X_train=X_train, y_train=y_train, **settings)
```
@@ -143,7 +143,8 @@ print('Predicted labels', y_pred)
#### Compute different metric values on testing dataset
```python
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
+
print('r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))
print('mse', '=', sklearn_metric_loss_score('mse', y_pred, y_test))
print('mae', '=', sklearn_metric_loss_score('mae', y_pred, y_test))
@@ -160,7 +161,8 @@ from xgboost import XGBRegressor
xgb = XGBRegressor()
xgb.fit(X_train, y_train)
y_pred = xgb.predict(X_test)
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
+
print('default xgboost r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))
# default xgboost r2 = 0.8265451174596482
```
@@ -170,10 +172,10 @@ print('default xgboost r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_t
How does the model accuracy improve as we search for different hyperparameter configurations?
```python
-from flaml.data import get_output_from_log
+from flaml.automl.data import get_output_from_log
import numpy as np
-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \
+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =
get_output_from_log(filename=settings['log_file_name'], time_budget=60)
plt.title('Learning Curve')
plt.xlabel('Wall Clock Time (s)')
@@ -190,15 +192,18 @@ You can easily enable a custom objective function by adding a customized XGBoost
```python
import numpy as np
+
# define your customized objective function
def logregobj(preds, dtrain):
labels = dtrain.get_label()
- preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight
+ preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight
grad = preds - labels
hess = preds * (1.0 - preds)
return grad, hess
-from flaml.model import XGBoostEstimator
+
+from flaml.automl.model import XGBoostEstimator
+
class MyXGB1(XGBoostEstimator):
'''XGBoostEstimator with the logregobj function as the objective function
@@ -207,6 +212,7 @@ class MyXGB1(XGBoostEstimator):
def __init__(self, **config):
super().__init__(objective=logregobj, **config)
+
class MyXGB2(XGBoostEstimator):
'''XGBoostEstimator with 'reg:squarederror' as the objective function
'''
diff --git a/website/docs/Examples/Default-Flamlized.md b/website/docs/Examples/Default-Flamlized.md
index edc9b08b7..d87ee47f9 100644
--- a/website/docs/Examples/Default-Flamlized.md
+++ b/website/docs/Examples/Default-Flamlized.md
@@ -9,9 +9,9 @@ This example requires openml==0.10.2.
### Zero-shot AutoML
```python
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml.default import LGBMRegressor
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")
lgbm = LGBMRegressor()
@@ -65,9 +65,9 @@ X_test.shape: (5160, 8), y_test.shape: (5160,)
### Zero-shot AutoML
```python
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml.default import XGBClassifier
-from flaml.ml import sklearn_metric_loss_score
+from flaml.automl.ml import sklearn_metric_loss_score
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
xgb = XGBClassifier()
diff --git a/website/docs/Examples/Integrate - AzureML.md b/website/docs/Examples/Integrate - AzureML.md
index 6a7e6a3c6..4d9db9088 100644
--- a/website/docs/Examples/Integrate - AzureML.md
+++ b/website/docs/Examples/Integrate - AzureML.md
@@ -27,7 +27,7 @@ mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())
### Start an AutoML run
```python
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml import AutoML
# Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure.
diff --git a/website/docs/Examples/Integrate - Scikit-learn Pipeline.md b/website/docs/Examples/Integrate - Scikit-learn Pipeline.md
index 7f272cea7..ee32ebd0d 100644
--- a/website/docs/Examples/Integrate - Scikit-learn Pipeline.md
+++ b/website/docs/Examples/Integrate - Scikit-learn Pipeline.md
@@ -5,7 +5,7 @@ This example requires openml==0.10.2.
### Load data
```python
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
# Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure.
X_train, X_test, y_train, y_test = load_openml_dataset(
diff --git a/website/docs/Getting-Started.md b/website/docs/Getting-Started.md
index 0c54e2d45..e3e828cbc 100644
--- a/website/docs/Getting-Started.md
+++ b/website/docs/Getting-Started.md
@@ -44,7 +44,8 @@ You can run generic hyperparameter tuning for a custom function (machine learnin
```python
from flaml import tune
-from flaml.model import LGBMEstimator
+from flaml.automl.model import LGBMEstimator
+
def train_lgbm(config: dict) -> dict:
# convert config dict to lgbm params
@@ -58,6 +59,7 @@ def train_lgbm(config: dict) -> dict:
# return eval results as a dictionary
return {"mse": mse}
+
# load a built-in search space from flaml
flaml_lgbm_search_space = LGBMEstimator.search_space(X_train.shape)
# specify the search space as a dict from hp name to domain; you can define your own search space same way
@@ -92,7 +94,7 @@ Then, you can use it just like you use the original `LGMBClassifier`. Your other
* Find code examples under "Examples": from [AutoML - Classification](Examples/AutoML-Classification) to [Tune - PyTorch](Examples/Tune-PyTorch).
* Find [talks](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A) and [tutorials](https://github.com/microsoft/FLAML/tree/tutorial/tutorial) about FLAML.
* Learn about [research](Research) around FLAML.
-* Refer to [SDK](reference/automl) and [FAQ](FAQ).
+* Refer to [SDK](reference/automl/automl) and [FAQ](FAQ).
If you like our project, please give it a [star](https://github.com/microsoft/FLAML/stargazers) on GitHub. If you are interested in contributing, please read [Contributor's Guide](Contribute).
diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md
index 7627a510d..94025e57e 100644
--- a/website/docs/Use-Cases/Task-Oriented-AutoML.md
+++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md
@@ -2,7 +2,7 @@
## Overview
-[`flaml.AutoML`](../reference/automl#automl-objects) is a class for task-oriented AutoML. It can be used as a scikit-learn style estimator with the standard `fit` and `predict` functions. The minimal inputs from users are the training data and the task type.
+[`flaml.AutoML`](../reference/automl/automl#automl-objects) is a class for task-oriented AutoML. It can be used as a scikit-learn style estimator with the standard `fit` and `predict` functions. The minimal inputs from users are the training data and the task type.
* Training data:
- numpy array. When the input data are stored in numpy array, they are passed to `fit()` as `X_train` and `y_train`.
@@ -134,48 +134,49 @@ The estimator list can contain one or more estimator names, each corresponding t
#### Guidelines on tuning a custom estimator
To tune a custom estimator that is not built-in, you need to:
-1. Build a custom estimator by inheritting [`flaml.model.BaseEstimator`](../reference/model#baseestimator-objects) or a derived class.
+1. Build a custom estimator by inheritting [`flaml.model.BaseEstimator`](../reference/automl/model#baseestimator-objects) or a derived class.
For example, if you have a estimator class with scikit-learn style `fit()` and `predict()` functions, you only need to set `self.estimator_class` to be that class in your constructor.
```python
-from flaml.model import SKLearnEstimator
+from flaml.automl.model import SKLearnEstimator
# SKLearnEstimator is derived from BaseEstimator
import rgf
-class MyRegularizedGreedyForest(SKLearnEstimator):
- def __init__(self, task="binary", **config):
- super().__init__(task, **config)
-
- if task in CLASSIFICATION:
- from rgf.sklearn import RGFClassifier
-
- self.estimator_class = RGFClassifier
- else:
- from rgf.sklearn import RGFRegressor
-
- self.estimator_class = RGFRegressor
- @classmethod
- def search_space(cls, data_size, task):
- space = {
- "max_leaf": {
- "domain": tune.lograndint(lower=4, upper=data_size),
- "low_cost_init_value": 4,
- },
- "n_iter": {
- "domain": tune.lograndint(lower=1, upper=data_size),
- "low_cost_init_value": 1,
- },
- "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},
- "min_samples_leaf": {
- "domain": tune.lograndint(lower=1, upper=20),
- "init_value": 20,
- },
- }
- return space
+class MyRegularizedGreedyForest(SKLearnEstimator):
+ def __init__(self, task="binary", **config):
+ super().__init__(task, **config)
+
+ if task in CLASSIFICATION:
+ from rgf.sklearn import RGFClassifier
+
+ self.estimator_class = RGFClassifier
+ else:
+ from rgf.sklearn import RGFRegressor
+
+ self.estimator_class = RGFRegressor
+
+ @classmethod
+ def search_space(cls, data_size, task):
+ space = {
+ "max_leaf": {
+ "domain": tune.lograndint(lower=4, upper=data_size),
+ "low_cost_init_value": 4,
+ },
+ "n_iter": {
+ "domain": tune.lograndint(lower=1, upper=data_size),
+ "low_cost_init_value": 1,
+ },
+ "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},
+ "min_samples_leaf": {
+ "domain": tune.lograndint(lower=1, upper=20),
+ "init_value": 20,
+ },
+ }
+ return space
```
-In the constructor, we set `self.estimator_class` as `RGFClassifier` or `RGFRegressor` according to the task type. If the estimator you want to tune does not have a scikit-learn style `fit()` and `predict()` API, you can override the `fit()` and `predict()` function of `flaml.model.BaseEstimator`, like [XGBoostEstimator](../reference/model#xgboostestimator-objects). Importantly, we also add the `task="binary"` parameter in the signature of `__init__` so that it doesn't get grouped together with the `**config` kwargs that determines the parameters with which the underlying estimator (`self.estimator_class`) is constructed. If your estimator doesn't use one of the parameters that it is passed, for example some regressors in `scikit-learn` don't use the `n_jobs` parameter, it is enough to add `n_jobs=None` to the signature so that it is ignored by the `**config` dict.
+In the constructor, we set `self.estimator_class` as `RGFClassifier` or `RGFRegressor` according to the task type. If the estimator you want to tune does not have a scikit-learn style `fit()` and `predict()` API, you can override the `fit()` and `predict()` function of `flaml.model.BaseEstimator`, like [XGBoostEstimator](../reference/automl/model#xgboostestimator-objects). Importantly, we also add the `task="binary"` parameter in the signature of `__init__` so that it doesn't get grouped together with the `**config` kwargs that determines the parameters with which the underlying estimator (`self.estimator_class`) is constructed. If your estimator doesn't use one of the parameters that it is passed, for example some regressors in `scikit-learn` don't use the `n_jobs` parameter, it is enough to add `n_jobs=None` to the signature so that it is ignored by the `**config` dict.
2. Give the custom estimator a name and add it in AutoML. E.g.,
@@ -203,20 +204,22 @@ In the example above, we tune four hyperparameters, three integers and one float
To customize the search space for a built-in estimator, use a similar approach to define a class that inherits the existing estimator. For example,
```python
-from flaml.model import XGBoostEstimator
+from flaml.automl.model import XGBoostEstimator
+
def logregobj(preds, dtrain):
- labels = dtrain.get_label()
- preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight
- grad = preds - labels
- hess = preds * (1.0 - preds)
- return grad, hess
+ labels = dtrain.get_label()
+ preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight
+ grad = preds - labels
+ hess = preds * (1.0 - preds)
+ return grad, hess
+
class MyXGB1(XGBoostEstimator):
- """XGBoostEstimator with logregobj as the objective function"""
+ """XGBoostEstimator with logregobj as the objective function"""
- def __init__(self, **config):
- super().__init__(objective=logregobj, **config)
+ def __init__(self, **config):
+ super().__init__(objective=logregobj, **config)
```
We override the constructor and set the training objective as a custom function `logregobj`. The hyperparameters and their search range do not change. For another example,
@@ -242,7 +245,7 @@ We override the `search_space` function to tune two hyperparameters only, "n_est
##### A shortcut to override the search space
-One can use the `custom_hp` argument in [`AutoML.fit()`](../reference/automl#fit) to override the search space for an existing estimator quickly. For example, if you would like to temporarily change the search range of "n_estimators" of xgboost, disable searching "max_leaves" in random forest, and add "subsample" in the search space of lightgbm, you can set:
+One can use the `custom_hp` argument in [`AutoML.fit()`](../reference/automl/automl#fit) to override the search space for an existing estimator quickly. For example, if you would like to temporarily change the search range of "n_estimators" of xgboost, disable searching "max_leaves" in random forest, and add "subsample" in the search space of lightgbm, you can set:
```python
custom_hp = {
@@ -456,7 +459,7 @@ Extra fit arguments that are needed by the estimators can be passed to `AutoML.f
In addition, you can specify the different arguments needed by different estimators using the `fit_kwargs_by_estimator` argument. For example, you can set the custom arguments for a Transformers model:
```python
-from flaml.data import load_openml_dataset
+from flaml.automl.data import load_openml_dataset
from flaml import AutoML
X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
@@ -487,7 +490,7 @@ print(automl.model)
# <flaml.model.LGBMEstimator object at 0x7f9b502c4550>
```
-[`flaml.model.LGBMEstimator`](../reference/model#lgbmestimator-objects) is a wrapper class for LightGBM models. To access the underlying model, use the `estimator` property of the `flaml.model.LGBMEstimator` instance.
+[`flaml.model.LGBMEstimator`](../reference/automl/model#lgbmestimator-objects) is a wrapper class for LightGBM models. To access the underlying model, use the `estimator` property of the `flaml.model.LGBMEstimator` instance.
```python
print(automl.model.estimator)
@@ -547,9 +550,9 @@ print(automl.config_history)
To plot how the loss is improved over time during the model search, first load the search history from the log file:
```python
-from flaml.data import get_output_from_log
+from flaml.automl.data import get_output_from_log
-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \
+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =
get_output_from_log(filename=settings["log_file_name"], time_budget=120)
```
diff --git a/website/docs/Use-Cases/Tune-User-Defined-Function.md b/website/docs/Use-Cases/Tune-User-Defined-Function.md
index 9aca4cf62..39f04eb86 100644
--- a/website/docs/Use-Cases/Tune-User-Defined-Function.md
+++ b/website/docs/Use-Cases/Tune-User-Defined-Function.md
@@ -339,7 +339,8 @@ In the following code example, we consider the sample size as the resource dimen
```python
from flaml import tune
from functools import partial
-from flaml.data import load_openml_task
+from flaml.automl.data import load_openml_task
+
def obj_from_resource_attr(resource_attr, X_train, X_test, y_train, y_test, config):
from lightgbm import LGBMClassifier
@@ -362,13 +363,14 @@ def obj_from_resource_attr(resource_attr, X_train, X_test, y_train, y_test, conf
test_loss = 1.0 - accuracy_score(y_test, y_test_predict)
return {resource_attr: resource, "loss": test_loss}
+
X_train, X_test, y_train, y_test = load_openml_task(task_id=7592, data_dir="test/")
max_resource = len(y_train)
resource_attr = "sample_size"
min_resource = 1000
analysis = tune.run(
partial(obj_from_resource_attr, resource_attr, X_train, X_test, y_train, y_test),
- config = {
+ config={
"n_estimators": tune.lograndint(lower=4, upper=32768),
"max_leaves": tune.lograndint(lower=4, upper=32768),
"learning_rate": tune.loguniform(lower=1 / 1024, upper=1.0),
diff --git a/website/docs/Use-Cases/Zero-Shot-AutoML.md b/website/docs/Use-Cases/Zero-Shot-AutoML.md
index c8871a3fd..bbda22566 100644
--- a/website/docs/Use-Cases/Zero-Shot-AutoML.md
+++ b/website/docs/Use-Cases/Zero-Shot-AutoML.md
@@ -1,6 +1,6 @@
# Zero Shot AutoML
-`flaml.default` is a package for zero-shot AutoML, or "no-tuning" AutoML. It uses [`flaml.AutoML`](../reference/automl#automl-objects) and [`flaml.default.portfolio`](../reference/default/portfolio) to mine good hyperparameter configurations across different datasets offline, and recommend data-dependent default configurations at runtime without expensive tuning.
+`flaml.default` is a package for zero-shot AutoML, or "no-tuning" AutoML. It uses [`flaml.AutoML`](../reference/automl/automl#automl-objects) and [`flaml.default.portfolio`](../reference/default/portfolio) to mine good hyperparameter configurations across different datasets offline, and recommend data-dependent default configurations at runtime without expensive tuning.
Zero-shot AutoML has several benefits:
* The computation cost is just training one model. No tuning is involved.
@@ -48,6 +48,7 @@ Yes. You can use `suggest_hyperparams()` to find the suggested configuration. Fo
```python
from flaml.default import LGBMRegressor
+
estimator = LGBMRegressor()
hyperparams, estimator_name, X_transformed, y_transformed = estimator.suggest_hyperparams(X_train, y_train)
print(hyperparams)
@@ -57,6 +58,7 @@ If you would like more control over the training, use an equivalent, open-box wa
```python
from flaml.default import preprocess_and_suggest_hyperparams
+
X, y = load_iris(return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
hyperparams, estimator_class, X_transformed, y_transformed, feature_transformer, label_transformer = preprocess_and_suggest_hyperparams(
@@ -239,6 +241,7 @@ Optionally, you can "flamlize" a learner using [`flaml.default.flamlize_estimato
```python
import sklearn.ensemble as ensemble
from flaml.default import flamlize_estimator
+
ExtraTreesClassifier = flamlize_estimator(
ensemble.ExtraTreesClassifier, "extra_tree", "classification"
)
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index 227ea0559..fc47c8d44 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -28,7 +28,7 @@ module.exports = {
},
{
type: 'doc',
- docId: 'reference/automl',
+ docId: 'reference/automl/automl',
position: 'left',
label: 'SDK',
},