A programming framework for agentic AI
Refactor into automl subpackage (#809)
Mark Harley44ddf9e10465
diff --git a/README.md b/README.mdindex 9290aa329..d2590a6cc 100644--- a/README.md+++ b/README.md@@ -84,6 +84,7 @@ tune.run(evaluation_function, config={…}, low_cost_partial_config={…}, time_```pythonfrom flaml.default import LGBMRegressor+# Use LGBMRegressor in the same way as you use lightgbm.LGBMRegressor.estimator = LGBMRegressor()# The hyperparameters are automatically set according to the training data.diff --git a/flaml/__init__.py b/flaml/__init__.pyindex ecd80cf69..161a81cea 100644--- a/flaml/__init__.py+++ b/flaml/__init__.py@@ -1,5 +1,5 @@-from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearchfrom flaml.automl import AutoML, logger_formatter+from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearchfrom flaml.onlineml.autovw import AutoVWfrom flaml.version import __version__import loggingdiff --git a/flaml/automl/__init__.py b/flaml/automl/__init__.pynew file mode 100644index 000000000..b93cd5b14--- /dev/null+++ b/flaml/automl/__init__.py@@ -0,0 +1,3 @@+from flaml.automl.automl import AutoML, AutoMLState, SearchState, logger_formatter, size++__all__ = ["AutoML", "AutoMLState", "SearchState", "logger_formatter", "size"]diff --git a/flaml/automl.py b/flaml/automl/automl.pysimilarity index 99%rename from flaml/automl.pyrename to flaml/automl/automl.pyindex 664e712cf..794bb1748 100644--- a/flaml/automl.py+++ b/flaml/automl/automl.py@@ -22,13 +22,13 @@ from sklearn.base import BaseEstimatorimport pandas as pdimport loggingimport json-from .ml import (+from flaml.automl.ml import (compute_estimator,train_estimator,get_estimator_class,get_classification_objective,)-from .config import (+from flaml.config import (MIN_SAMPLE_TRAIN,MEM_THRES,RANDOM_SEED,@@ -38,7 +38,7 @@ from .config import (N_SPLITS,SAMPLE_MULTIPLY_FACTOR,)-from .data import (+from flaml.automl.data import (concat,CLASSIFICATION,TOKENCLASSIFICATION,@@ -50,9 +50,10 @@ from .data import (_is_nlp_task,NLG_TASKS,)-from . import tune-from .training_log import training_log_reader, training_log_writer-from flaml.default.suggest import suggest_learner+from flaml import tune+from flaml.automl.training_log import training_log_reader, training_log_writer+from flaml.default import suggest_learner+from flaml.version import __version__ as flaml_versionlogger = logging.getLogger(__name__)logger_formatter = logging.Formatter(@@ -78,7 +79,7 @@ class SearchState:)def valid_starting_point_one_dim(self, value_one_dim, domain_one_dim):- from .tune.space import sample+ from flaml.tune.space import sample"""For each hp in the starting point, check the following 3 conditions:@@ -500,7 +501,7 @@ class AutoML(BaseEstimator):"""- from .version import __version__+ __version__ = flaml_versiondef __init__(self, **settings):"""Constructor.@@ -640,7 +641,7 @@ class AutoML(BaseEstimator):seed: int or None, default=None | The random seed for hpo.n_concurrent_trials: [Experimental] int, default=1 | The number ofconcurrent trials. When n_concurrent_trials > 1, flaml performes- [parallel tuning](../Use-Cases/Task-Oriented-AutoML#parallel-tuning)+ [parallel tuning](../../Use-Cases/Task-Oriented-AutoML#parallel-tuning)and installation of ray is required: `pip install flaml[ray]`.keep_search_state: boolean, default=False | Whether to keep data neededfor model search after fit(). By default the state is deleted for@@ -671,7 +672,7 @@ class AutoML(BaseEstimator):the metrics_to_log dictionary returned by a customized metric function.The customized metric function shall be provided via the `metric` key wordargument of the fit() function or the automl constructor.- Find an example in the 4th constraint type in this [doc](../Use-Cases/Task-Oriented-AutoML#constraint).+ Find an example in the 4th constraint type in this [doc](../../Use-Cases/Task-Oriented-AutoML#constraint).If `pred_time_limit` is provided as one of keyword arguments to fit() function orthe automl constructor, flaml will automatically (and under the hood)add it as an additional element in the metric_constraints. Essentially 'pred_time_limit'@@ -2336,7 +2337,7 @@ class AutoML(BaseEstimator):seed: int or None, default=None | The random seed for hpo.n_concurrent_trials: [Experimental] int, default=1 | The number ofconcurrent trials. When n_concurrent_trials > 1, flaml performes- [parallel tuning](../Use-Cases/Task-Oriented-AutoML#parallel-tuning)+ [parallel tuning](../../Use-Cases/Task-Oriented-AutoML#parallel-tuning)and installation of ray is required: `pip install flaml[ray]`.keep_search_state: boolean, default=False | Whether to keep data neededfor model search after fit(). By default the state is deleted for@@ -2699,7 +2700,9 @@ class AutoML(BaseEstimator):)if "auto" == metric:if _is_nlp_task(self._state.task):- from .nlp.utils import load_default_huggingface_metric_for_task+ from flaml.automl.nlp.utils import (+ load_default_huggingface_metric_for_task,+ )metric = load_default_huggingface_metric_for_task(self._state.task)elif "binary" in self._state.task:@@ -2734,7 +2737,7 @@ class AutoML(BaseEstimator):]:return True, f"1-{metric}"if _is_nlp_task(task):- from .ml import huggingface_metric_to_mode+ from flaml.automl.ml import huggingface_metric_to_modeif (metric in huggingface_metric_to_modediff --git a/flaml/automl/data.py b/flaml/automl/data.pynew file mode 100644index 000000000..01f27d14e--- /dev/null+++ b/flaml/automl/data.py@@ -0,0 +1,485 @@+# !+# * Copyright (c) Microsoft Corporation. All rights reserved.+# * Licensed under the MIT License. See LICENSE file in the+# * project root for license information.+import numpy as np+from scipy.sparse import vstack, issparse+import pandas as pd+from pandas import DataFrame, Series++from flaml.automl.training_log import training_log_reader++from datetime import datetime+from typing import Union++# TODO: if your task is not specified in here, define your task as an all-capitalized word+SEQCLASSIFICATION = "seq-classification"+MULTICHOICECLASSIFICATION = "multichoice-classification"+TOKENCLASSIFICATION = "token-classification"+CLASSIFICATION = (+ "binary",+ "multiclass",+ "classification",+ SEQCLASSIFICATION,+ MULTICHOICECLASSIFICATION,+ TOKENCLASSIFICATION,+)+SEQREGRESSION = "seq-regression"+REGRESSION = ("regression", SEQREGRESSION)+TS_FORECASTREGRESSION = (+ "forecast",+ "ts_forecast",+ "ts_forecast_regression",+)+TS_FORECASTCLASSIFICATION = "ts_forecast_classification"+TS_FORECASTPANEL = "ts_forecast_panel"+TS_FORECAST = (+ *TS_FORECASTREGRESSION,+ TS_FORECASTCLASSIFICATION,+ TS_FORECASTPANEL,+)+TS_TIMESTAMP_COL = "ds"+TS_VALUE_COL = "y"+SUMMARIZATION = "summarization"+NLG_TASKS = (SUMMARIZATION,)+NLU_TASKS = (+ SEQREGRESSION,+ SEQCLASSIFICATION,+ MULTICHOICECLASSIFICATION,+ TOKENCLASSIFICATION,+)+++def _is_nlp_task(task):+ if task in NLU_TASKS or task in NLG_TASKS:+ return True+ else:+ return False+++def load_openml_dataset(+ dataset_id, data_dir=None, random_state=0, dataset_format="dataframe"+):+ """Load dataset from open ML.++ If the file is not cached locally, download it from open ML.++ Args:+ dataset_id: An integer of the dataset id in openml.+ data_dir: A string of the path to store and load the data.+ random_state: An integer of the random seed for splitting data.+ dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'.+ Can choose from ['dataframe', 'array'].+ If 'dataframe', the returned dataset will be a Pandas DataFrame.+ If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix.++ Returns:+ X_train: Training data.+ X_test: Test data.+ y_train: A series or array of labels for training data.+ y_test: A series or array of labels for test data.+ """+ import os+ import openml+ import pickle+ from sklearn.model_selection import train_test_split++ filename = "openml_ds" + str(dataset_id) + ".pkl"+ filepath = os.path.join(data_dir, filename)+ if os.path.isfile(filepath):+ print("load dataset from", filepath)+ with open(filepath, "rb") as f:+ dataset = pickle.load(f)+ else:+ print("download dataset from openml")+ dataset = openml.datasets.get_dataset(dataset_id)+ if not os.path.exists(data_dir):+ os.makedirs(data_dir)+ with open(filepath, "wb") as f:+ pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)+ print("Dataset name:", dataset.name)+ try:+ X, y, *__ = dataset.get_data(+ target=dataset.default_target_attribute, dataset_format=dataset_format+ )+ except ValueError:+ from sklearn.datasets import fetch_openml++ X, y = fetch_openml(data_id=dataset_id, return_X_y=True)+ X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state)+ print(+ "X_train.shape: {}, y_train.shape: {};\nX_test.shape: {}, y_test.shape: {}".format(+ X_train.shape,+ y_train.shape,+ X_test.shape,+ y_test.shape,+ )+ )+ return X_train, X_test, y_train, y_test+++def load_openml_task(task_id, data_dir):+ """Load task from open ML.++ Use the first fold of the task.+ If the file is not cached locally, download it from open ML.++ Args:+ task_id: An integer of the task id in openml.+ data_dir: A string of the path to store and load the data.++ Returns:+ X_train: A dataframe of training data.+ X_test: A dataframe of test data.+ y_train: A series of labels for training data.+ y_test: A series of labels for test data.+ """+ import os+ import openml+ import pickle++ task = openml.tasks.get_task(task_id)+ filename = "openml_task" + str(task_id) + ".pkl"+ filepath = os.path.join(data_dir, filename)+ if os.path.isfile(filepath):+ print("load dataset from", filepath)+ with open(filepath, "rb") as f:+ dataset = pickle.load(f)+ else:+ print("download dataset from openml")+ dataset = task.get_dataset()+ with open(filepath, "wb") as f:+ pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)+ X, y, _, _ = dataset.get_data(task.target_name)+ train_indices, test_indices = task.get_train_test_split_indices(+ repeat=0,+ fold=0,+ sample=0,+ )+ X_train = X.iloc[train_indices]+ y_train = y[train_indices]+ X_test = X.iloc[test_indices]+ y_test = y[test_indices]+ print(+ "X_train.shape: {}, y_train.shape: {},\nX_test.shape: {}, y_test.shape: {}".format(+ X_train.shape,+ y_train.shape,+ X_test.shape,+ y_test.shape,+ )+ )+ return X_train, X_test, y_train, y_test+++def get_output_from_log(filename, time_budget):+ """Get output from log file.++ Args:+ filename: A string of the log file name.+ time_budget: A float of the time budget in seconds.++ Returns:+ search_time_list: A list of the finished time of each logged iter.+ best_error_list: A list of the best validation error after each logged iter.+ error_list: A list of the validation error of each logged iter.+ config_list: A list of the estimator, sample size and config of each logged iter.+ logged_metric_list: A list of the logged metric of each logged iter.+ """++ best_config = None+ best_learner = None+ best_val_loss = float("+inf")++ search_time_list = []+ config_list = []+ best_error_list = []+ error_list = []+ logged_metric_list = []+ best_config_list = []+ with training_log_reader(filename) as reader:+ for record in reader.records():+ time_used = record.wall_clock_time+ val_loss = record.validation_loss+ config = record.config+ learner = record.learner.split("_")[0]+ sample_size = record.sample_size+ metric = record.logged_metric++ if time_used < time_budget and np.isfinite(val_loss):+ if val_loss < best_val_loss:+ best_val_loss = val_loss+ best_config = config+ best_learner = learner+ best_config_list.append(best_config)+ search_time_list.append(time_used)+ best_error_list.append(best_val_loss)+ logged_metric_list.append(metric)+ error_list.append(val_loss)+ config_list.append(+ {+ "Current Learner": learner,+ "Current Sample": sample_size,+ "Current Hyper-parameters": record.config,+ "Best Learner": best_learner,+ "Best Hyper-parameters": best_config,+ }+ )++ return (+ search_time_list,+ best_error_list,+ error_list,+ config_list,+ logged_metric_list,+ )+++def concat(X1, X2):+ """concatenate two matrices vertically."""+ if isinstance(X1, (DataFrame, Series)):+ df = pd.concat([X1, X2], sort=False)+ df.reset_index(drop=True, inplace=True)+ if isinstance(X1, DataFrame):+ cat_columns = X1.select_dtypes(include="category").columns+ if len(cat_columns):+ df[cat_columns] = df[cat_columns].astype("category")+ return df+ if issparse(X1):+ return vstack((X1, X2))+ else:+ return np.concatenate([X1, X2])+++def add_time_idx_col(X):+ unique_dates = X[TS_TIMESTAMP_COL].drop_duplicates().sort_values(ascending=True)+ # assume no missing timestamps+ freq = pd.infer_freq(unique_dates)+ if freq == "MS":+ X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year * 12 + X[TS_TIMESTAMP_COL].dt.month+ elif freq == "Y":+ X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year+ else:+ # using time frequency to generate all time stamps and then indexing for time_idx+ # full_range = pd.date_range(X[TS_TIMESTAMP_COL].min(), X[TS_TIMESTAMP_COL].max(), freq=freq).to_list()+ # X["time_idx"] = [full_range.index(time) for time in X[TS_TIMESTAMP_COL]]+ # taking minimum difference in timestamp+ timestamps = unique_dates.view("int64")+ freq = int(timestamps.diff().mode())+ X["time_idx"] = timestamps - timestamps.min() / freq+ X["time_idx"] = X["time_idx"].astype("int")+ return X+++class DataTransformer:+ """Transform input training data."""++ def fit_transform(self, X: Union[DataFrame, np.array], y, task):+ """Fit transformer and process the input training data according to the task type.++ Args:+ X: A numpy array or a pandas dataframe of training data.+ y: A numpy array or a pandas series of labels.+ task: A string of the task type, e.g.,+ 'classification', 'regression', 'ts_forecast', 'rank'.++ Returns:+ X: Processed numpy array or pandas dataframe of training data.+ y: Processed numpy array or pandas series of labels.+ """+ if _is_nlp_task(task):+ # if the mode is NLP, check the type of input, each column must be either string or+ # ids (input ids, token type id, attention mask, etc.)+ str_columns = []+ for column in X.columns:+ if isinstance(X[column].iloc[0], str):+ str_columns.append(column)+ if len(str_columns) > 0:+ X[str_columns] = X[str_columns].astype("string")+ self._str_columns = str_columns+ elif isinstance(X, DataFrame):+ X = X.copy()+ n = X.shape[0]+ cat_columns, num_columns, datetime_columns = [], [], []+ drop = False+ if task in TS_FORECAST:+ X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})+ if task is TS_FORECASTPANEL:+ if "time_idx" not in X:+ X = add_time_idx_col(X)+ ds_col = X.pop(TS_TIMESTAMP_COL)+ if isinstance(y, Series):+ y = y.rename(TS_VALUE_COL)+ for column in X.columns:+ # sklearn\utils\validation.py needs int/float values+ if X[column].dtype.name in ("object", "category"):+ if (+ X[column].nunique() == 1+ or X[column].nunique(dropna=True)+ == n - X[column].isnull().sum()+ ):+ X.drop(columns=column, inplace=True)+ drop = True+ elif X[column].dtype.name == "category":+ current_categories = X[column].cat.categories+ if "__NAN__" not in current_categories:+ X[column] = (+ X[column]+ .cat.add_categories("__NAN__")+ .fillna("__NAN__")+ )+ cat_columns.append(column)+ else:+ X[column] = X[column].fillna("__NAN__")+ cat_columns.append(column)+ elif X[column].nunique(dropna=True) < 2:+ X.drop(columns=column, inplace=True)+ drop = True+ else: # datetime or numeric+ if X[column].dtype.name == "datetime64[ns]":+ tmp_dt = X[column].dt+ new_columns_dict = {+ f"year_{column}": tmp_dt.year,+ f"month_{column}": tmp_dt.month,+ f"day_{column}": tmp_dt.day,+ f"hour_{column}": tmp_dt.hour,+ f"minute_{column}": tmp_dt.minute,+ f"second_{column}": tmp_dt.second,+ f"dayofweek_{column}": tmp_dt.dayofweek,+ f"dayofyear_{column}": tmp_dt.dayofyear,+ f"quarter_{column}": tmp_dt.quarter,+ }+ for key, value in new_columns_dict.items():+ if (+ key not in X.columns+ and value.nunique(dropna=False) >= 2+ ):+ X[key] = value+ num_columns.append(key)+ X[column] = X[column].map(datetime.toordinal)+ datetime_columns.append(column)+ del tmp_dt+ X[column] = X[column].fillna(np.nan)+ num_columns.append(column)+ X = X[cat_columns + num_columns]+ if task in TS_FORECAST:+ X.insert(0, TS_TIMESTAMP_COL, ds_col)+ if cat_columns:+ X[cat_columns] = X[cat_columns].astype("category")+ if num_columns:+ X_num = X[num_columns]+ if np.issubdtype(X_num.columns.dtype, np.integer) and (+ drop+ or min(X_num.columns) != 0+ or max(X_num.columns) != X_num.shape[1] - 1+ ):+ X_num.columns = range(X_num.shape[1])+ drop = True+ else:+ drop = False+ from sklearn.impute import SimpleImputer+ from sklearn.compose import ColumnTransformer++ self.transformer = ColumnTransformer(+ [+ (+ "continuous",+ SimpleImputer(missing_values=np.nan, strategy="median"),+ X_num.columns,+ )+ ]+ )+ X[num_columns] = self.transformer.fit_transform(X_num)+ self._cat_columns, self._num_columns, self._datetime_columns = (+ cat_columns,+ num_columns,+ datetime_columns,+ )+ self._drop = drop+ if (+ task in CLASSIFICATION+ or not pd.api.types.is_numeric_dtype(y)+ and task not in NLG_TASKS+ ):+ if task != TOKENCLASSIFICATION:+ from sklearn.preprocessing import LabelEncoder++ self.label_transformer = LabelEncoder()+ else:+ from flaml.automl.nlp.utils import LabelEncoderforTokenClassification++ self.label_transformer = LabelEncoderforTokenClassification()+ y = self.label_transformer.fit_transform(y)++ else:+ self.label_transformer = None+ self._task = task+ return X, y++ def transform(self, X: Union[DataFrame, np.array]):+ """Process data using fit transformer.++ Args:+ X: A numpy array or a pandas dataframe of training data.++ Returns:+ X: Processed numpy array or pandas dataframe of training data.+ """+ X = X.copy()++ if _is_nlp_task(self._task):+ # if the mode is NLP, check the type of input, each column must be either string or+ # ids (input ids, token type id, attention mask, etc.)+ if len(self._str_columns) > 0:+ X[self._str_columns] = X[self._str_columns].astype("string")+ elif isinstance(X, DataFrame):+ cat_columns, num_columns, datetime_columns = (+ self._cat_columns,+ self._num_columns,+ self._datetime_columns,+ )+ if self._task in TS_FORECAST:+ X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})+ ds_col = X.pop(TS_TIMESTAMP_COL)+ for column in datetime_columns:+ tmp_dt = X[column].dt+ new_columns_dict = {+ f"year_{column}": tmp_dt.year,+ f"month_{column}": tmp_dt.month,+ f"day_{column}": tmp_dt.day,+ f"hour_{column}": tmp_dt.hour,+ f"minute_{column}": tmp_dt.minute,+ f"second_{column}": tmp_dt.second,+ f"dayofweek_{column}": tmp_dt.dayofweek,+ f"dayofyear_{column}": tmp_dt.dayofyear,+ f"quarter_{column}": tmp_dt.quarter,+ }+ for new_col_name, new_col_value in new_columns_dict.items():+ if new_col_name not in X.columns and new_col_name in num_columns:+ X[new_col_name] = new_col_value+ X[column] = X[column].map(datetime.toordinal)+ del tmp_dt+ X = X[cat_columns + num_columns].copy()+ if self._task in TS_FORECAST:+ X.insert(0, TS_TIMESTAMP_COL, ds_col)+ for column in cat_columns:+ if X[column].dtype.name == "object":+ X[column] = X[column].fillna("__NAN__")+ elif X[column].dtype.name == "category":+ current_categories = X[column].cat.categories+ if "__NAN__" not in current_categories:+ X[column] = (+ X[column].cat.add_categories("__NAN__").fillna("__NAN__")+ )+ if cat_columns:+ X[cat_columns] = X[cat_columns].astype("category")+ if num_columns:+ X_num = X[num_columns].fillna(np.nan)+ if self._drop:+ X_num.columns = range(X_num.shape[1])+ X[num_columns] = self.transformer.transform(X_num)+ return X+++def group_counts(groups):+ _, i, c = np.unique(groups, return_counts=True, return_index=True)+ return c[np.argsort(i)]diff --git a/flaml/automl/ml.py b/flaml/automl/ml.pynew file mode 100644index 000000000..6285bc29c--- /dev/null+++ b/flaml/automl/ml.py@@ -0,0 +1,749 @@+# !+# * Copyright (c) FLAML authors. All rights reserved.+# * Licensed under the MIT License. See LICENSE file in the+# * project root for license information.+import time+import numpy as np+import pandas as pd+from sklearn.metrics import (+ mean_squared_error,+ r2_score,+ roc_auc_score,+ accuracy_score,+ mean_absolute_error,+ log_loss,+ average_precision_score,+ f1_score,+ mean_absolute_percentage_error,+ ndcg_score,+)+from sklearn.model_selection import RepeatedStratifiedKFold, GroupKFold, TimeSeriesSplit+from flaml.automl.model import (+ XGBoostSklearnEstimator,+ XGBoost_TS,+ XGBoostLimitDepthEstimator,+ XGBoostLimitDepth_TS,+ RandomForestEstimator,+ RF_TS,+ LGBMEstimator,+ LGBM_TS,+ LRL1Classifier,+ LRL2Classifier,+ CatBoostEstimator,+ ExtraTreesEstimator,+ ExtraTrees_TS,+ KNeighborsEstimator,+ Prophet,+ ARIMA,+ SARIMAX,+ TransformersEstimator,+ TemporalFusionTransformerEstimator,+ TransformersEstimatorModelSelection,+)+from flaml.automl.data import CLASSIFICATION, group_counts, TS_FORECAST+import logging++logger = logging.getLogger(__name__)++sklearn_metric_name_set = {+ "r2",+ "rmse",+ "mae",+ "mse",+ "accuracy",+ "roc_auc",+ "roc_auc_ovr",+ "roc_auc_ovo",+ "roc_auc_weighted",+ "roc_auc_ovr_weighted",+ "roc_auc_ovo_weighted",+ "log_loss",+ "mape",+ "f1",+ "ap",+ "ndcg",+ "micro_f1",+ "macro_f1",+}+huggingface_metric_to_mode = {+ "accuracy": "max",+ "bertscore": "max",+ "bleu": "max",+ "bleurt": "max",+ "cer": "min",+ "chrf": "min",+ "code_eval": "max",+ "comet": "max",+ "competition_math": "max",+ "coval": "max",+ "cuad": "max",+ "f1": "max",+ "gleu": "max",+ "google_bleu": "max",+ "matthews_correlation": "max",+ "meteor": "max",+ "pearsonr": "max",+ "precision": "max",+ "recall": "max",+ "rouge": "max",+ "sacrebleu": "max",+ "sari": "max",+ "seqeval": "max",+ "spearmanr": "max",+ "ter": "min",+ "wer": "min",+}+huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}+++def get_estimator_class(task, estimator_name):+ # when adding a new learner, need to add an elif branch+ if "xgboost" == estimator_name:+ estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator+ elif "xgb_limitdepth" == estimator_name:+ estimator_class = (+ XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator+ )+ elif "rf" == estimator_name:+ estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator+ elif "lgbm" == estimator_name:+ estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator+ elif "lrl1" == estimator_name:+ estimator_class = LRL1Classifier+ elif "lrl2" == estimator_name:+ estimator_class = LRL2Classifier+ elif "catboost" == estimator_name:+ estimator_class = CatBoostEstimator+ elif "extra_tree" == estimator_name:+ estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator+ elif "kneighbor" == estimator_name:+ estimator_class = KNeighborsEstimator+ elif "prophet" in estimator_name:+ estimator_class = Prophet+ elif estimator_name == "arima":+ estimator_class = ARIMA+ elif estimator_name == "sarimax":+ estimator_class = SARIMAX+ elif estimator_name == "transformer":+ estimator_class = TransformersEstimator+ elif estimator_name == "tft":+ estimator_class = TemporalFusionTransformerEstimator+ elif estimator_name == "transformer_ms":+ estimator_class = TransformersEstimatorModelSelection+ else:+ raise ValueError(+ estimator_name + " is not a built-in learner. "+ "Please use AutoML.add_learner() to add a customized learner."+ )+ return estimator_class+++def metric_loss_score(+ metric_name,+ y_processed_predict,+ y_processed_true,+ labels=None,+ sample_weight=None,+ groups=None,+):+ # y_processed_predict and y_processed_true are processed id labels if the original were the token labels+ if is_in_sklearn_metric_name_set(metric_name):+ return sklearn_metric_loss_score(+ metric_name,+ y_processed_predict,+ y_processed_true,+ labels,+ sample_weight,+ groups,+ )+ else:+ try:+ import datasets++ datasets_metric_name = huggingface_submetric_to_metric.get(+ metric_name, metric_name.split(":")[0]+ )+ metric = datasets.load_metric(datasets_metric_name)+ metric_mode = huggingface_metric_to_mode[datasets_metric_name]++ if metric_name.startswith("seqeval"):+ y_processed_true = [+ [labels[tr] for tr in each_list] for each_list in y_processed_true+ ]+ elif metric in ("pearsonr", "spearmanr"):+ y_processed_true = (+ y_processed_true.to_list()+ if isinstance(y_processed_true, pd.Series)+ else list(y_processed_true)+ )+ score_dict = metric.compute(+ predictions=y_processed_predict, references=y_processed_true+ )+ if "rouge" in metric_name:+ score = score_dict[metric_name].mid.fmeasure+ elif metric_name.startswith("seqeval"):+ metric_submetric_names = metric_name.split(":")+ score = score_dict[+ metric_submetric_names[1]+ if len(metric_submetric_names) > 1+ else "overall_accuracy"+ ]+ else:+ score = score_dict[metric_name]+ except ImportError:+ raise ValueError(+ metric_name+ + " is not an built-in sklearn metric and nlp is not installed. "+ "Currently built-in sklearn metrics are: "+ "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"+ "log_loss, mape, f1, micro_f1, macro_f1, ap. "+ "If the metric is an nlp metric, please pip install flaml[nlp] ",+ "or pass a customized metric function to AutoML.fit(metric=func)",+ )+ # If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)+ # ask the user to provide a custom metric+ except FileNotFoundError:+ raise ValueError(+ metric_name + " is neither an sklearn metric nor a huggingface metric. "+ "Currently built-in sklearn metrics are: "+ "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"+ "log_loss, mape, f1, micro_f1, macro_f1, ap. "+ "Currently built-in huggingface metrics are: "+ + ", ".join(huggingface_metric_to_mode.keys())+ + ". Please pass a customized metric function to AutoML.fit(metric=func)"+ )+ if metric_mode == "max":+ return 1 - score+ else:+ return score+++def is_in_sklearn_metric_name_set(metric_name):+ return metric_name.startswith("ndcg") or metric_name in sklearn_metric_name_set+++def is_min_metric(metric_name):+ return (+ metric_name in ["rmse", "mae", "mse", "log_loss", "mape"]+ or huggingface_metric_to_mode.get(metric_name, None) == "min"+ )+++def sklearn_metric_loss_score(+ metric_name,+ y_predict,+ y_true,+ labels=None,+ sample_weight=None,+ groups=None,+):+ """Loss using the specified metric.++ Args:+ metric_name: A string of the metric name, one of+ 'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'roc_auc_ovr',+ 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted',+ 'log_loss', 'mape', 'f1', 'ap', 'ndcg', 'micro_f1', 'macro_f1'.+ y_predict: A 1d or 2d numpy array of the predictions which can be+ used to calculate the metric. E.g., 2d for log_loss and 1d+ for others.+ y_true: A 1d numpy array of the true labels.+ labels: A list or an array of the unique labels.+ sample_weight: A 1d numpy array of the sample weight.+ groups: A 1d numpy array of the group labels.++ Returns:+ score: A float number of the loss, the lower the better.+ """+ metric_name = metric_name.lower()++ if "r2" == metric_name:+ score = 1.0 - r2_score(y_true, y_predict, sample_weight=sample_weight)+ elif metric_name == "rmse":+ score = np.sqrt(+ mean_squared_error(y_true, y_predict, sample_weight=sample_weight)+ )+ elif metric_name == "mae":+ score = mean_absolute_error(y_true, y_predict, sample_weight=sample_weight)+ elif metric_name == "mse":+ score = mean_squared_error(y_true, y_predict, sample_weight=sample_weight)+ elif metric_name == "accuracy":+ score = 1.0 - accuracy_score(y_true, y_predict, sample_weight=sample_weight)+ elif metric_name == "roc_auc":+ score = 1.0 - roc_auc_score(y_true, y_predict, sample_weight=sample_weight)+ elif metric_name == "roc_auc_ovr":+ score = 1.0 - roc_auc_score(+ y_true, y_predict, sample_weight=sample_weight, multi_class="ovr"+ )+ elif metric_name == "roc_auc_ovo":+ score = 1.0 - roc_auc_score(+ y_true, y_predict, sample_weight=sample_weight, multi_class="ovo"+ )+ elif metric_name == "roc_auc_weighted":+ score = 1.0 - roc_auc_score(+ y_true, y_predict, sample_weight=sample_weight, average="weighted"+ )+ elif metric_name == "roc_auc_ovo_weighted":+ score = 1.0 - roc_auc_score(+ y_true,+ y_predict,+ sample_weight=sample_weight,+ average="weighted",+ multi_class="ovo",+ )+ elif metric_name == "roc_auc_ovr_weighted":+ score = 1.0 - roc_auc_score(+ y_true,+ y_predict,+ sample_weight=sample_weight,+ average="weighted",+ multi_class="ovr",+ )+ elif "log_loss" == metric_name:+ score = log_loss(y_true, y_predict, labels=labels, sample_weight=sample_weight)+ elif "mape" == metric_name:+ try:+ score = mean_absolute_percentage_error(y_true, y_predict)+ except ValueError:+ return np.inf+ elif "micro_f1" == metric_name:+ score = 1 - f1_score(+ y_true, y_predict, sample_weight=sample_weight, average="micro"+ )+ elif "macro_f1" == metric_name:+ score = 1 - f1_score(+ y_true, y_predict, sample_weight=sample_weight, average="macro"+ )+ elif "f1" == metric_name:+ score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight)+ elif "ap" == metric_name:+ score = 1 - average_precision_score(+ y_true, y_predict, sample_weight=sample_weight+ )+ elif "ndcg" in metric_name:+ if "@" in metric_name:+ k = int(metric_name.split("@", 1)[-1])+ counts = group_counts(groups)+ score = 0+ psum = 0+ for c in counts:+ score -= ndcg_score(+ np.asarray([y_true[psum : psum + c]]),+ np.asarray([y_predict[psum : psum + c]]),+ k=k,+ )+ psum += c+ score /= len(counts)+ score += 1+ else:+ score = 1 - ndcg_score([y_true], [y_predict])+ return score+++def get_y_pred(estimator, X, eval_metric, obj):+ if eval_metric in ["roc_auc", "ap", "roc_auc_weighted"] and "binary" in obj:+ y_pred_classes = estimator.predict_proba(X)+ y_pred = y_pred_classes[:, 1] if y_pred_classes.ndim > 1 else y_pred_classes+ elif eval_metric in [+ "log_loss",+ "roc_auc",+ "roc_auc_ovr",+ "roc_auc_ovo",+ "roc_auc_ovo_weighted",+ "roc_auc_ovr_weighted",+ ]:+ y_pred = estimator.predict_proba(X)+ else:+ y_pred = estimator.predict(X)+ return y_pred+++def _eval_estimator(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ obj,+ labels=None,+ log_training_metric=False,+ fit_kwargs={},+):+ if isinstance(eval_metric, str):+ pred_start = time.time()+ val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)+ pred_time = (time.time() - pred_start) / X_val.shape[0]++ val_loss = metric_loss_score(+ eval_metric,+ y_processed_predict=val_pred_y,+ y_processed_true=y_val,+ labels=labels,+ sample_weight=weight_val,+ groups=groups_val,+ )+ metric_for_logging = {"pred_time": pred_time}+ if log_training_metric:+ train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj)+ metric_for_logging["train_loss"] = metric_loss_score(+ eval_metric,+ train_pred_y,+ y_train,+ labels,+ fit_kwargs.get("sample_weight"),+ fit_kwargs.get("groups"),+ )+ else: # customized metric function+ val_loss, metric_for_logging = eval_metric(+ X_val,+ y_val,+ estimator,+ labels,+ X_train,+ y_train,+ weight_val,+ fit_kwargs.get("sample_weight"),+ config,+ groups_val,+ fit_kwargs.get("groups"),+ )+ pred_time = metric_for_logging.get("pred_time", 0)+ val_pred_y = None+ # eval_metric may return val_pred_y but not necessarily. Setting None for now.+ return val_loss, metric_for_logging, pred_time, val_pred_y+++def get_val_loss(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ obj,+ labels=None,+ budget=None,+ log_training_metric=False,+ fit_kwargs={},+ free_mem_ratio=0,+):++ start = time.time()+ # if groups_val is not None:+ # fit_kwargs['groups_val'] = groups_val+ # fit_kwargs['X_val'] = X_val+ # fit_kwargs['y_val'] = y_val+ estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)+ val_loss, metric_for_logging, pred_time, _ = _eval_estimator(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ obj,+ labels,+ log_training_metric,+ fit_kwargs,+ )+ if hasattr(estimator, "intermediate_results"):+ metric_for_logging["intermediate_results"] = estimator.intermediate_results+ train_time = time.time() - start+ return val_loss, metric_for_logging, train_time, pred_time+++def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):+ metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)+ metrics_to_log = None+ for single_fold in log_metrics_folds:+ if metrics_to_log is None:+ metrics_to_log = single_fold+ elif isinstance(metrics_to_log, dict):+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}+ else:+ metrics_to_log += single_fold+ if metrics_to_log:+ n = len(val_loss_folds)+ metrics_to_log = (+ {k: v / n for k, v in metrics_to_log.items()}+ if isinstance(metrics_to_log, dict)+ else metrics_to_log / n+ )+ return metric_to_minimize, metrics_to_log+++def evaluate_model_CV(+ config,+ estimator,+ X_train_all,+ y_train_all,+ budget,+ kf,+ task,+ eval_metric,+ best_val_loss,+ cv_score_agg_func=None,+ log_training_metric=False,+ fit_kwargs={},+ free_mem_ratio=0,+):+ if cv_score_agg_func is None:+ cv_score_agg_func = default_cv_score_agg_func+ start_time = time.time()+ val_loss_folds = []+ log_metric_folds = []+ metric = None+ train_time = pred_time = 0+ total_fold_num = 0+ n = kf.get_n_splits()+ X_train_split, y_train_split = X_train_all, y_train_all+ if task in CLASSIFICATION:+ labels = np.unique(y_train_all)+ else:+ labels = fit_kwargs.get(+ "label_list"+ ) # pass the label list on to compute the evaluation metric+ groups = None+ shuffle = getattr(kf, "shuffle", task not in TS_FORECAST)+ if isinstance(kf, RepeatedStratifiedKFold):+ kf = kf.split(X_train_split, y_train_split)+ elif isinstance(kf, GroupKFold):+ groups = kf.groups+ kf = kf.split(X_train_split, y_train_split, groups)+ shuffle = False+ elif isinstance(kf, TimeSeriesSplit):+ kf = kf.split(X_train_split, y_train_split)+ else:+ kf = kf.split(X_train_split)+ rng = np.random.RandomState(2020)+ budget_per_train = budget and budget / n+ if "sample_weight" in fit_kwargs:+ weight = fit_kwargs["sample_weight"]+ weight_val = None+ else:+ weight = weight_val = None+ for train_index, val_index in kf:+ if shuffle:+ train_index = rng.permutation(train_index)+ if isinstance(X_train_all, pd.DataFrame):+ X_train = X_train_split.iloc[train_index]+ X_val = X_train_split.iloc[val_index]+ else:+ X_train, X_val = X_train_split[train_index], X_train_split[val_index]+ y_train, y_val = y_train_split[train_index], y_train_split[val_index]+ estimator.cleanup()+ if weight is not None:+ fit_kwargs["sample_weight"], weight_val = (+ weight[train_index],+ weight[val_index],+ )+ if groups is not None:+ fit_kwargs["groups"] = groups[train_index]+ groups_val = groups[val_index]+ else:+ groups_val = None+ val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ task,+ labels,+ budget_per_train,+ log_training_metric=log_training_metric,+ fit_kwargs=fit_kwargs,+ free_mem_ratio=free_mem_ratio,+ )+ if isinstance(metric_i, dict) and "intermediate_results" in metric_i.keys():+ del metric_i["intermediate_results"]+ if weight is not None:+ fit_kwargs["sample_weight"] = weight+ total_fold_num += 1+ val_loss_folds.append(val_loss_i)+ log_metric_folds.append(metric_i)+ train_time += train_time_i+ pred_time += pred_time_i+ if budget and time.time() - start_time >= budget:+ break+ val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)+ n = total_fold_num+ pred_time /= n+ return val_loss, metric, train_time, pred_time+++def compute_estimator(+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ budget,+ kf,+ config_dic,+ task,+ estimator_name,+ eval_method,+ eval_metric,+ best_val_loss=np.Inf,+ n_jobs=1,+ estimator_class=None,+ cv_score_agg_func=None,+ log_training_metric=False,+ fit_kwargs={},+ free_mem_ratio=0,+):+ estimator_class = estimator_class or get_estimator_class(task, estimator_name)+ estimator = estimator_class(+ **config_dic,+ task=task,+ n_jobs=n_jobs,+ )++ if isinstance(estimator, TransformersEstimator):+ # TODO: move the partial function to nlp+ fit_kwargs["metric"] = eval_metric+ fit_kwargs["X_val"] = X_val+ fit_kwargs["y_val"] = y_val++ if "holdout" == eval_method:+ val_loss, metric_for_logging, train_time, pred_time = get_val_loss(+ config_dic,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ task,+ labels=fit_kwargs.get(+ "label_list"+ ), # pass the label list on to compute the evaluation metric+ budget=budget,+ log_training_metric=log_training_metric,+ fit_kwargs=fit_kwargs,+ free_mem_ratio=0,+ )+ else:+ val_loss, metric_for_logging, train_time, pred_time = evaluate_model_CV(+ config_dic,+ estimator,+ X_train,+ y_train,+ budget,+ kf,+ task,+ eval_metric,+ best_val_loss,+ cv_score_agg_func,+ log_training_metric=log_training_metric,+ fit_kwargs=fit_kwargs,+ free_mem_ratio=0,+ )++ if isinstance(estimator, TransformersEstimator):+ del fit_kwargs["metric"], fit_kwargs["X_val"], fit_kwargs["y_val"]++ return estimator, val_loss, metric_for_logging, train_time, pred_time+++def train_estimator(+ config_dic,+ X_train,+ y_train,+ task,+ estimator_name,+ n_jobs=1,+ estimator_class=None,+ budget=None,+ fit_kwargs={},+ eval_metric=None,+ free_mem_ratio=0,+):+ start_time = time.time()+ estimator_class = estimator_class or get_estimator_class(task, estimator_name)+ estimator = estimator_class(+ **config_dic,+ task=task,+ n_jobs=n_jobs,+ )+ if isinstance(estimator, TransformersEstimator):+ fit_kwargs["metric"] = eval_metric++ if X_train is not None:+ train_time = estimator.fit(+ X_train, y_train, budget, free_mem_ratio, **fit_kwargs+ )+ else:+ estimator = estimator.estimator_class(**estimator.params)+ train_time = time.time() - start_time+ return estimator, train_time+++def get_classification_objective(num_labels: int) -> str:+ if num_labels == 2:+ objective_name = "binary"+ else:+ objective_name = "multiclass"+ return objective_name+++def norm_confusion_matrix(y_true, y_pred):+ """normalized confusion matrix.++ Args:+ estimator: A multi-class classification estimator.+ y_true: A numpy array or a pandas series of true labels.+ y_pred: A numpy array or a pandas series of predicted labels.++ Returns:+ A normalized confusion matrix.+ """+ from sklearn.metrics import confusion_matrix++ conf_mat = confusion_matrix(y_true, y_pred)+ norm_conf_mat = conf_mat.astype("float") / conf_mat.sum(axis=1)[:, np.newaxis]+ return norm_conf_mat+++def multi_class_curves(y_true, y_pred_proba, curve_func):+ """Binarize the data for multi-class tasks and produce ROC or precision-recall curves.++ Args:+ y_true: A numpy array or a pandas series of true labels.+ y_pred_proba: A numpy array or a pandas dataframe of predicted probabilites.+ curve_func: A function to produce a curve (e.g., roc_curve or precision_recall_curve).++ Returns:+ A tuple of two dictionaries with the same set of keys (class indices).+ The first dictionary curve_x stores the x coordinates of each curve, e.g.,+ curve_x[0] is an 1D array of the x coordinates of class 0.+ The second dictionary curve_y stores the y coordinates of each curve, e.g.,+ curve_y[0] is an 1D array of the y coordinates of class 0.+ """+ from sklearn.preprocessing import label_binarize++ classes = np.unique(y_true)+ y_true_binary = label_binarize(y_true, classes=classes)++ curve_x, curve_y = {}, {}+ for i in range(len(classes)):+ curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])+ return curve_x, curve_ydiff --git a/flaml/automl/model.py b/flaml/automl/model.pynew file mode 100644index 000000000..bccad6470--- /dev/null+++ b/flaml/automl/model.py@@ -0,0 +1,2393 @@+# !+# * Copyright (c) FLAML authors. All rights reserved.+# * Licensed under the MIT License. See LICENSE file in the+# * project root for license information.+from contextlib import contextmanager+from functools import partial+import signal+import os+from typing import Callable, List+import numpy as np+import time+from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier+from sklearn.ensemble import ExtraTreesRegressor, ExtraTreesClassifier+from sklearn.linear_model import LogisticRegression+from sklearn.dummy import DummyClassifier, DummyRegressor+from scipy.sparse import issparse+import logging+import shutil+from pandas import DataFrame, Series, to_datetime+import sys+import math+from flaml import tune+from flaml.automl.data import (+ group_counts,+ CLASSIFICATION,+ add_time_idx_col,+ TS_FORECASTREGRESSION,+ TS_TIMESTAMP_COL,+ TS_VALUE_COL,+ SEQCLASSIFICATION,+ SEQREGRESSION,+ TOKENCLASSIFICATION,+ SUMMARIZATION,+ NLG_TASKS,+)++try:+ import psutil+except ImportError:+ psutil = None+try:+ import resource+except ImportError:+ resource = None++logger = logging.getLogger("flaml.automl")+# FREE_MEM_RATIO = 0.2+++def TimeoutHandler(sig, frame):+ raise TimeoutError(sig, frame)+++@contextmanager+def limit_resource(memory_limit, time_limit):+ if memory_limit > 0:+ soft, hard = resource.getrlimit(resource.RLIMIT_AS)+ if soft < 0 and (hard < 0 or memory_limit <= hard) or memory_limit < soft:+ try:+ resource.setrlimit(resource.RLIMIT_AS, (int(memory_limit), hard))+ except ValueError:+ # According to https://bugs.python.org/issue40518, it's a mac-specific error.+ pass+ main_thread = False+ if time_limit is not None:+ try:+ signal.signal(signal.SIGALRM, TimeoutHandler)+ signal.alarm(int(time_limit) or 1)+ main_thread = True+ except ValueError:+ pass+ try:+ yield+ finally:+ if main_thread:+ signal.alarm(0)+ if memory_limit > 0:+ resource.setrlimit(resource.RLIMIT_AS, (soft, hard))+++class BaseEstimator:+ """The abstract class for all learners.++ Typical examples:+ * XGBoostEstimator: for regression.+ * XGBoostSklearnEstimator: for classification.+ * LGBMEstimator, RandomForestEstimator, LRL1Classifier, LRL2Classifier:+ for both regression and classification.+ """++ def __init__(self, task="binary", **config):+ """Constructor.++ Args:+ task: A string of the task type, one of+ 'binary', 'multiclass', 'regression', 'rank', 'seq-classification',+ 'seq-regression', 'token-classification', 'multichoice-classification',+ 'summarization', 'ts_forecast', 'ts_forecast_classification'.+ config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.+ n_jobs is the number of parallel threads.+ """+ self._task = task+ self.params = self.config2params(config)+ self.estimator_class = self._model = None+ if "_estimator_type" in config:+ self._estimator_type = self.params.pop("_estimator_type")+ else:+ self._estimator_type = (+ "classifier" if task in CLASSIFICATION else "regressor"+ )++ def get_params(self, deep=False):+ params = self.params.copy()+ params["task"] = self._task+ if hasattr(self, "_estimator_type"):+ params["_estimator_type"] = self._estimator_type+ return params++ @property+ def classes_(self):+ return self._model.classes_++ @property+ def n_features_in_(self):+ return self._model.n_features_in_++ @property+ def model(self):+ """Trained model after fit() is called, or None before fit() is called."""+ return self._model++ @property+ def estimator(self):+ """Trained model after fit() is called, or None before fit() is called."""+ return self._model++ @property+ def feature_names_in_(self):+ """+ if self._model has attribute feature_names_in_, return it.+ otherwise, if self._model has attribute feature_name_, return it.+ otherwise, if self._model has attribute feature_names, return it.+ otherwise, if self._model has method get_booster, return the feature names.+ otherwise, return None.+ """+ if hasattr(self._model, "feature_names_in_"): # for sklearn, xgboost>=1.6+ return self._model.feature_names_in_+ if hasattr(self._model, "feature_name_"): # for lightgbm+ return self._model.feature_name_+ if hasattr(self._model, "feature_names"): # for XGBoostEstimator+ return self._model.feature_names+ if hasattr(self._model, "get_booster"):+ # get feature names for xgboost<1.6+ # https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.Booster.feature_names+ booster = self._model.get_booster()+ return booster.feature_names+ return None++ @property+ def feature_importances_(self):+ """+ if self._model has attribute feature_importances_, return it.+ otherwise, if self._model has attribute coef_, return it.+ otherwise, return None.+ """+ if hasattr(self._model, "feature_importances_"):+ # for sklearn, lightgbm, catboost, xgboost+ return self._model.feature_importances_+ elif hasattr(self._model, "coef_"): # for linear models+ return self._model.coef_+ else:+ return None++ def _preprocess(self, X):+ return X++ def _fit(self, X_train, y_train, **kwargs):++ current_time = time.time()+ if "groups" in kwargs:+ kwargs = kwargs.copy()+ groups = kwargs.pop("groups")+ if self._task == "rank":+ kwargs["group"] = group_counts(groups)+ # groups_val = kwargs.get('groups_val')+ # if groups_val is not None:+ # kwargs['eval_group'] = [group_counts(groups_val)]+ # kwargs['eval_set'] = [+ # (kwargs['X_val'], kwargs['y_val'])]+ # kwargs['verbose'] = False+ # del kwargs['groups_val'], kwargs['X_val'], kwargs['y_val']+ X_train = self._preprocess(X_train)+ model = self.estimator_class(**self.params)+ if logger.level == logging.DEBUG:+ # xgboost 1.6 doesn't display all the params in the model str+ logger.debug(f"flaml.model - {model} fit started with params {self.params}")+ model.fit(X_train, y_train, **kwargs)+ if logger.level == logging.DEBUG:+ logger.debug(f"flaml.model - {model} fit finished")+ train_time = time.time() - current_time+ self._model = model+ return train_time++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ """Train the model from given training data.++ Args:+ X_train: A numpy array or a dataframe of training data in shape n*m.+ y_train: A numpy array or a series of labels in shape n*1.+ budget: A float of the time budget in seconds.+ free_mem_ratio: A float between 0 and 1 for the free memory ratio to keep during training.++ Returns:+ train_time: A float of the training time in seconds.+ """+ if (+ getattr(self, "limit_resource", None)+ and resource is not None+ and (budget is not None or psutil is not None)+ ):+ start_time = time.time()+ mem = psutil.virtual_memory() if psutil is not None else None+ try:+ with limit_resource(+ mem.available * (1 - free_mem_ratio)+ + psutil.Process(os.getpid()).memory_info().rss+ if mem is not None+ else -1,+ budget,+ ):+ train_time = self._fit(X_train, y_train, **kwargs)+ except (MemoryError, TimeoutError) as e:+ logger.warning(f"{e.__class__} {e}")+ if self._task in CLASSIFICATION:+ model = DummyClassifier()+ else:+ model = DummyRegressor()+ X_train = self._preprocess(X_train)+ model.fit(X_train, y_train)+ self._model = model+ train_time = time.time() - start_time+ else:+ train_time = self._fit(X_train, y_train, **kwargs)+ return train_time++ def predict(self, X, **kwargs):+ """Predict label from features.++ Args:+ X: A numpy array or a dataframe of featurized instances, shape n*m.++ Returns:+ A numpy array of shape n*1.+ Each element is the label for a instance.+ """+ if self._model is not None:+ X = self._preprocess(X)+ return self._model.predict(X, **kwargs)+ else:+ logger.warning(+ "Estimator is not fit yet. Please run fit() before predict()."+ )+ return np.ones(X.shape[0])++ def predict_proba(self, X, **kwargs):+ """Predict the probability of each class from features.++ Only works for classification problems++ Args:+ X: A numpy array of featurized instances, shape n*m.++ Returns:+ A numpy array of shape n*c. c is the # classes.+ Each element at (i,j) is the probability for instance i to be in+ class j.+ """+ assert self._task in CLASSIFICATION, "predict_proba() only for classification."++ X = self._preprocess(X)+ return self._model.predict_proba(X, **kwargs)++ def score(self, X_val: DataFrame, y_val: Series, **kwargs):+ """Report the evaluation score of a trained estimator.+++ Args:+ X_val: A pandas dataframe of the validation input data.+ y_val: A pandas series of the validation label.+ kwargs: keyword argument of the evaluation function, for example:+ - metric: A string of the metric name or a function+ e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',+ 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',+ 'mape'. Default is 'auto'.+ If metric is given, the score will report the user specified metric.+ If metric is not given, the metric is set to accuracy for classification and r2+ for regression.+ You can also pass a customized metric function, for examples on how to pass a+ customized metric function, please check+ [test/nlp/test_autohf_custom_metric.py](https://github.com/microsoft/FLAML/blob/main/test/nlp/test_autohf_custom_metric.py) and+ [test/automl/test_multiclass.py](https://github.com/microsoft/FLAML/blob/main/test/automl/test_multiclass.py).++ Returns:+ The evaluation score on the validation dataset.+ """+ from .ml import metric_loss_score+ from .ml import is_min_metric++ if self._model is not None:+ if self._task == "rank":+ raise NotImplementedError(+ "AutoML.score() is not implemented for ranking"+ )+ else:+ X_val = self._preprocess(X_val)+ metric = kwargs.pop("metric", None)+ if metric:+ y_pred = self.predict(X_val, **kwargs)+ if is_min_metric(metric):+ return metric_loss_score(metric, y_pred, y_val)+ else:+ return 1.0 - metric_loss_score(metric, y_pred, y_val)+ else:+ return self._model.score(X_val, y_val, **kwargs)+ else:+ logger.warning(+ "Estimator is not fit yet. Please run fit() before predict()."+ )+ return 0.0++ def cleanup(self):+ del self._model+ self._model = None++ @classmethod+ def search_space(cls, data_size, task, **params):+ """[required method] search space.++ Args:+ data_size: A tuple of two integers, number of rows and columns.+ task: A str of the task type, e.g., "binary", "multiclass", "regression".++ Returns:+ A dictionary of the search space.+ Each key is the name of a hyperparameter, and value is a dict with+ its domain (required) and low_cost_init_value, init_value,+ cat_hp_cost (if applicable).+ e.g., ```{'domain': tune.randint(lower=1, upper=10), 'init_value': 1}```.+ """+ return {}++ @classmethod+ def size(cls, config: dict) -> float:+ """[optional method] memory size of the estimator in bytes.++ Args:+ config: A dict of the hyperparameter config.++ Returns:+ A float of the memory size required by the estimator to train the+ given config.+ """+ return 1.0++ @classmethod+ def cost_relative2lgbm(cls) -> float:+ """[optional method] relative cost compared to lightgbm."""+ return 1.0++ @classmethod+ def init(cls):+ """[optional method] initialize the class."""+ pass++ def config2params(self, config: dict) -> dict:+ """[optional method] config dict to params dict++ Args:+ config: A dict of the hyperparameter config.++ Returns:+ A dict that will be passed to self.estimator_class's constructor.+ """+ params = config.copy()+ if "FLAML_sample_size" in params:+ params.pop("FLAML_sample_size")+ return params+++class TransformersEstimator(BaseEstimator):+ """The class for fine-tuning language models, using huggingface transformers API."""++ ITER_HP = "global_max_steps"++ def __init__(self, task="seq-classification", **config):+ super().__init__(task, **config)+ import uuid++ self.trial_id = str(uuid.uuid1().hex)[:8]+ if task not in NLG_TASKS: # TODO: not in NLG_TASKS+ from .nlp.huggingface.training_args import (+ TrainingArgumentsForAuto as TrainingArguments,+ )+ else:+ from .nlp.huggingface.training_args import (+ Seq2SeqTrainingArgumentsForAuto as TrainingArguments,+ )+ self._TrainingArguments = TrainingArguments++ @classmethod+ def search_space(cls, data_size, task, **params):+ search_space_dict = {+ "learning_rate": {+ "domain": tune.loguniform(1e-6, 1e-4),+ "init_value": 1e-5,+ },+ "num_train_epochs": {+ "domain": tune.choice([1, 2, 3, 4, 5]),+ "init_value": 3, # to be consistent with roberta+ "low_cost_init_value": 1,+ },+ "per_device_train_batch_size": {+ "domain": tune.choice([4, 8, 16, 32, 64]),+ "init_value": 32,+ "low_cost_init_value": 64,+ },+ "seed": {+ "domain": tune.choice(range(1, 40)),+ "init_value": 20,+ },+ "global_max_steps": {+ "domain": sys.maxsize,+ "init_value": sys.maxsize,+ },+ }++ return search_space_dict++ @property+ def fp16(self):+ return self._kwargs.get("gpu_per_trial") and self._training_args.fp16++ @property+ def no_cuda(self):+ return not self._kwargs.get("gpu_per_trial")++ def _set_training_args(self, **kwargs):+ from .nlp.utils import date_str, Counter++ for (key, val) in kwargs.items():+ assert key not in self.params, (+ "Since {} is in the search space, it cannot exist in 'custom_fit_kwargs' at the same time."+ "If you need to fix the value of {} to {}, the only way is to add a single-value domain in the search "+ "space by adding:\n '{}': {{ 'domain': {} }} to 'custom_hp'. For example:"+ 'automl_settings["custom_hp"] = {{ "transformer": {{ "model_path": {{ "domain" : '+ '"google/electra-small-discriminator" }} }} }}'.format(+ key, key, val, key, val+ )+ )++ """+ If use has specified any custom args for TrainingArguments, update these arguments+ """+ self._training_args = self._TrainingArguments(**kwargs)++ """+ Update the attributes in TrainingArguments with self.params values+ """+ for key, val in self.params.items():+ if hasattr(self._training_args, key):+ setattr(self._training_args, key, val)++ """+ Update the attributes in TrainingArguments that depends on the values of self.params+ """+ local_dir = os.path.join(+ self._training_args.output_dir, "train_{}".format(date_str())+ )+ if self._use_ray is True:+ import ray++ self._training_args.output_dir = ray.tune.get_trial_dir()+ else:+ self._training_args.output_dir = Counter.get_trial_fold_name(+ local_dir, self.params, self.trial_id+ )++ self._training_args.fp16 = self.fp16+ self._training_args.no_cuda = self.no_cuda++ if (+ self._task == TOKENCLASSIFICATION+ and self._training_args.max_seq_length is not None+ ):+ logger.warning(+ "For token classification task, FLAML currently does not support customizing the max_seq_length, max_seq_length will be reset to None."+ )+ setattr(self._training_args, "max_seq_length", None)++ def _tokenize_text(self, X, y=None, **kwargs):+ from .nlp.huggingface.utils import tokenize_text+ from .nlp.utils import is_a_list_of_str++ is_str = str(X.dtypes[0]) in ("string", "str")+ is_list_of_str = is_a_list_of_str(X[list(X.keys())[0]].to_list()[0])++ if is_str or is_list_of_str:+ return tokenize_text(+ X=X,+ Y=y,+ task=self._task,+ hf_args=self._training_args,+ tokenizer=self.tokenizer,+ )+ else:+ return X, y++ def _model_init(self):+ from .nlp.huggingface.utils import load_model++ this_model = load_model(+ checkpoint_path=self._training_args.model_path,+ task=self._task,+ num_labels=self.num_labels,+ )+ return this_model++ def _preprocess_data(self, X, y):+ from datasets import Dataset++ processed_X, processed_y_df = self._tokenize_text(X=X, y=y, **self._kwargs)+ # convert y from pd.DataFrame back to pd.Series+ processed_y = processed_y_df.iloc[:, 0]++ processed_dataset = Dataset.from_pandas(processed_X.join(processed_y_df))++ return processed_dataset, processed_X, processed_y++ @property+ def num_labels(self):+ if self._task == SEQREGRESSION:+ return 1+ elif self._task == SEQCLASSIFICATION:+ return len(set(self._y_train))+ elif self._task == TOKENCLASSIFICATION:+ return len(self._training_args.label_list)+ else:+ return None++ @property+ def tokenizer(self):+ from transformers import AutoTokenizer++ if self._task == SUMMARIZATION:+ return AutoTokenizer.from_pretrained(+ pretrained_model_name_or_path=self._training_args.model_path,+ cache_dir=None,+ use_fast=True,+ revision="main",+ use_auth_token=None,+ )+ else:+ return AutoTokenizer.from_pretrained(+ self._training_args.model_path,+ use_fast=True,+ add_prefix_space=self._add_prefix_space,+ )++ @property+ def data_collator(self):+ from .nlp.huggingface.data_collator import task_to_datacollator_class++ data_collator_class = task_to_datacollator_class.get(self._task)++ if data_collator_class:+ kwargs = {+ "model": self._model_init(),+ # need to set model, or there's ValueError: Expected input batch_size (..) to match target batch_size (..)+ "label_pad_token_id": -100, # pad with token id -100+ "pad_to_multiple_of": 8,+ # pad to multiple of 8 because quote Transformers: "This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta)"+ "tokenizer": self.tokenizer,+ }++ for key in list(kwargs.keys()):+ if (+ key not in data_collator_class.__dict__.keys()+ and key != "tokenizer"+ ):+ del kwargs[key]+ return data_collator_class(**kwargs)+ else:+ return None++ def fit(+ self,+ X_train: DataFrame,+ y_train: Series,+ budget=None,+ free_mem_ratio=0,+ X_val=None,+ y_val=None,+ gpu_per_trial=None,+ metric=None,+ **kwargs,+ ):+ import transformers++ transformers.logging.set_verbosity_error()++ from transformers import TrainerCallback+ from transformers.trainer_utils import set_seed+ from .nlp.huggingface.trainer import TrainerForAuto++ try:+ from ray.tune import is_session_enabled++ self._use_ray = is_session_enabled()+ except ImportError:+ self._use_ray = False++ this_params = self.params+ self._kwargs = kwargs++ self._X_train, self._y_train = X_train, y_train+ self._set_training_args(**kwargs)+ self._add_prefix_space = (+ "roberta" in self._training_args.model_path+ ) # If using roberta model, must set add_prefix_space to True to avoid the assertion error at+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249++ train_dataset, self._X_train, self._y_train = self._preprocess_data(+ X_train, y_train+ )+ if X_val is not None:+ eval_dataset, self._X_val, self._y_val = self._preprocess_data(X_val, y_val)+ else:+ eval_dataset, self._X_val, self._y_val = None, None, None++ set_seed(self.params.get("seed", self._training_args.seed))+ self._metric = metric++ class EarlyStoppingCallbackForAuto(TrainerCallback):+ def on_train_begin(self, args, state, control, **callback_kwargs):+ self.train_begin_time = time.time()++ def on_step_begin(self, args, state, control, **callback_kwargs):+ self.step_begin_time = time.time()++ def on_step_end(self, args, state, control, **callback_kwargs):+ if state.global_step == 1:+ self.time_per_iter = time.time() - self.step_begin_time+ if (+ budget+ and (+ time.time() + self.time_per_iter+ > self.train_begin_time + budget+ )+ or state.global_step >= this_params[TransformersEstimator.ITER_HP]+ ):+ control.should_training_stop = True+ control.should_save = True+ control.should_evaluate = True+ return control++ def on_epoch_end(self, args, state, control, **callback_kwargs):+ if (+ control.should_training_stop+ or state.epoch + 1 >= args.num_train_epochs+ ):+ control.should_save = True+ control.should_evaluate = True++ self._trainer = TrainerForAuto(+ args=self._training_args,+ model_init=self._model_init,+ train_dataset=train_dataset,+ eval_dataset=eval_dataset,+ tokenizer=self.tokenizer,+ data_collator=self.data_collator,+ compute_metrics=self._compute_metrics_by_dataset_name,+ callbacks=[EarlyStoppingCallbackForAuto],+ )++ if self._task in NLG_TASKS:+ setattr(self._trainer, "_is_seq2seq", True)++ """+ When not using ray for tuning, set the limit of CUDA_VISIBLE_DEVICES to math.ceil(gpu_per_trial),+ so each estimator does not see all the GPUs+ """+ if gpu_per_trial is not None:+ tmp_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "")+ self._trainer.args._n_gpu = gpu_per_trial++ # if gpu_per_trial == 0:+ # os.environ["CUDA_VISIBLE_DEVICES"] = ""+ if tmp_cuda_visible_devices.count(",") != math.ceil(gpu_per_trial) - 1:+ os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(+ [str(x) for x in range(math.ceil(gpu_per_trial))]+ )++ import time++ start_time = time.time()+ self._trainer.train()++ if gpu_per_trial is not None:+ os.environ["CUDA_VISIBLE_DEVICES"] = tmp_cuda_visible_devices++ self.params[self.ITER_HP] = self._trainer.state.global_step++ self._checkpoint_path = self._select_checkpoint(self._trainer)+ self._ckpt_remains = list(self._trainer.ckpt_to_metric.keys())++ if hasattr(self._trainer, "intermediate_results"):+ self.intermediate_results = [+ x[1]+ for x in sorted(+ self._trainer.intermediate_results.items(), key=lambda x: x[0]+ )+ ]+ self._trainer = None++ return time.time() - start_time++ def _delete_one_ckpt(self, ckpt_location):+ if self._use_ray is False:+ try:+ shutil.rmtree(ckpt_location)+ except FileNotFoundError:+ logger.warning("checkpoint {} not found".format(ckpt_location))++ def cleanup(self):+ super().cleanup()+ if hasattr(self, "_ckpt_remains"):+ for each_ckpt in self._ckpt_remains:+ self._delete_one_ckpt(each_ckpt)++ def _select_checkpoint(self, trainer):+ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR++ if trainer.ckpt_to_metric:+ best_ckpt, _ = min(+ trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_automl_metric"]+ )+ best_ckpt_global_step = trainer.ckpt_to_global_step[best_ckpt]+ for each_ckpt in list(trainer.ckpt_to_metric):+ if each_ckpt != best_ckpt:+ del trainer.ckpt_to_metric[each_ckpt]+ del trainer.ckpt_to_global_step[each_ckpt]+ self._delete_one_ckpt(each_ckpt)+ else:+ best_ckpt_global_step = trainer.state.global_step+ best_ckpt = os.path.join(+ trainer.args.output_dir,+ f"{PREFIX_CHECKPOINT_DIR}-{best_ckpt_global_step}",+ )+ self.params[self.ITER_HP] = best_ckpt_global_step+ logger.debug(trainer.state.global_step)+ logger.debug(trainer.ckpt_to_global_step)+ return best_ckpt++ def _compute_metrics_by_dataset_name(self, eval_pred):+ # TODO: call self._metric(eval_pred, self)+ if isinstance(self._metric, str):+ from .ml import metric_loss_score+ from .nlp.huggingface.utils import postprocess_prediction_and_true++ predictions, y_true = eval_pred+ # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text+ processed_predictions, processed_y_true = postprocess_prediction_and_true(+ task=self._task,+ y_pred=predictions,+ tokenizer=self.tokenizer,+ hf_args=self._training_args,+ y_true=y_true,+ )+ metric_dict = {+ "automl_metric": metric_loss_score(+ metric_name=self._metric,+ y_processed_predict=processed_predictions,+ y_processed_true=processed_y_true,+ labels=self._training_args.label_list,+ )+ }+ else:+ # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)+ loss, metric_dict = self._metric(+ X_test=self._X_val,+ y_test=self._y_val,+ estimator=self,+ labels=None,+ X_train=self._X_train,+ y_train=self._y_train,+ )+ metric_dict["automl_metric"] = loss++ return metric_dict++ def _init_model_for_predict(self):+ from .nlp.huggingface.trainer import TrainerForAuto++ """+ Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistent+ with HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947+ """+ training_args = self._TrainingArguments(+ local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16+ )+ for key, val in self._training_args.__dict__.items():+ if key not in ("local_rank", "model_path", "fp16"):+ setattr(training_args, key, val)+ self._training_args = training_args++ new_trainer = TrainerForAuto(+ model=self._model_init(),+ args=self._training_args,+ data_collator=self.data_collator,+ compute_metrics=self._compute_metrics_by_dataset_name,+ )+ if self._task in NLG_TASKS:+ setattr(new_trainer, "_is_seq2seq", True)+ return new_trainer++ def predict_proba(self, X, **pred_kwargs):+ from datasets import Dataset++ if pred_kwargs:+ for key, val in pred_kwargs.items():+ setattr(self._training_args, key, val)++ assert (+ self._task in CLASSIFICATION+ ), "predict_proba() only for classification tasks."++ X_test, _ = self._tokenize_text(X, **self._kwargs)+ test_dataset = Dataset.from_pandas(X_test)++ new_trainer = self._init_model_for_predict()+ predictions = new_trainer.predict(test_dataset)+ return predictions.predictions++ def score(self, X_val: DataFrame, y_val: Series, **kwargs):+ import transformers++ transformers.logging.set_verbosity_error()++ self._metric = kwargs["metric"]++ eval_dataset, X_val, y_val = self._preprocess_data(X_val, y_val)++ new_trainer = self._init_model_for_predict()+ return new_trainer.evaluate(eval_dataset)++ def predict(self, X, **pred_kwargs):+ import transformers+ from datasets import Dataset+ from .nlp.huggingface.utils import postprocess_prediction_and_true++ transformers.logging.set_verbosity_error()++ if pred_kwargs:+ for key, val in pred_kwargs.items():+ setattr(self._training_args, key, val)++ X_test, _ = self._tokenize_text(X, **self._kwargs)+ test_dataset = Dataset.from_pandas(X_test)++ new_trainer = self._init_model_for_predict()++ if self._task not in NLG_TASKS:+ predictions = new_trainer.predict(test_dataset)+ else:+ predictions = new_trainer.predict(+ test_dataset,+ metric_key_prefix="predict",+ )+ post_y_pred, _ = postprocess_prediction_and_true(+ task=self._task,+ y_pred=predictions.predictions,+ tokenizer=self.tokenizer,+ hf_args=self._training_args,+ X=X,+ )+ return post_y_pred++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ params[TransformersEstimator.ITER_HP] = params.get(+ TransformersEstimator.ITER_HP, sys.maxsize+ )+ return params+++class TransformersEstimatorModelSelection(TransformersEstimator):+ def __init__(self, task="seq-classification", **config):+ super().__init__(task, **config)++ @classmethod+ def search_space(cls, data_size, task, **params):+ search_space_dict = TransformersEstimator.search_space(+ data_size, task, **params+ )++ """+ For model selection, use the same search space regardless of memory constraint+ If OOM, user should change the search space themselves+ """++ search_space_dict["model_path"] = {+ "domain": tune.choice(+ [+ "google/electra-base-discriminator",+ "bert-base-uncased",+ "roberta-base",+ "facebook/muppet-roberta-base",+ "google/electra-small-discriminator",+ ]+ ),+ "init_value": "facebook/muppet-roberta-base",+ }+ return search_space_dict+++class SKLearnEstimator(BaseEstimator):+ """+ The base class for tuning scikit-learn estimators.++ Subclasses can modify the function signature of ``__init__`` to+ ignore the values in ``config`` that are not relevant to the constructor+ of their underlying estimator. For example, some regressors in ``scikit-learn``+ don't accept the ``n_jobs`` parameter contained in ``config``. For these,+ one can add ``n_jobs=None,`` before ``**config`` to make sure ``config`` doesn't+ contain an ``n_jobs`` key.+ """++ def __init__(self, task="binary", **config):+ super().__init__(task, **config)++ def _preprocess(self, X):+ if isinstance(X, DataFrame):+ cat_columns = X.select_dtypes(include=["category"]).columns+ if not cat_columns.empty:+ X = X.copy()+ X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":+ # numpy array is not of numeric dtype+ X = DataFrame(X)+ for col in X.columns:+ if isinstance(X[col][0], str):+ X[col] = X[col].astype("category").cat.codes+ X = X.to_numpy()+ return X+++class LGBMEstimator(BaseEstimator):+ """The class for tuning LGBM, using sklearn API."""++ ITER_HP = "n_estimators"+ HAS_CALLBACK = True+ DEFAULT_ITER = 100++ @classmethod+ def search_space(cls, data_size, **params):+ upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower+ return {+ "n_estimators": {+ "domain": tune.lograndint(lower=4, upper=upper),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ "num_leaves": {+ "domain": tune.lograndint(lower=4, upper=upper),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ "min_child_samples": {+ "domain": tune.lograndint(lower=2, upper=2**7 + 1),+ "init_value": 20,+ },+ "learning_rate": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1.0),+ "init_value": 0.1,+ },+ "log_max_bin": { # log transformed with base 2+ "domain": tune.lograndint(lower=3, upper=11),+ "init_value": 8,+ },+ "colsample_bytree": {+ "domain": tune.uniform(lower=0.01, upper=1.0),+ "init_value": 1.0,+ },+ "reg_alpha": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),+ "init_value": 1 / 1024,+ },+ "reg_lambda": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),+ "init_value": 1.0,+ },+ }++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ if "log_max_bin" in params:+ params["max_bin"] = (1 << params.pop("log_max_bin")) - 1+ return params++ @classmethod+ def size(cls, config):+ num_leaves = int(+ round(+ config.get("num_leaves")+ or config.get("max_leaves")+ or 1 << config.get("max_depth", 16)+ )+ )+ n_estimators = int(round(config["n_estimators"]))+ return (num_leaves * 3 + (num_leaves - 1) * 4 + 1.0) * n_estimators * 8++ def __init__(self, task="binary", **config):+ super().__init__(task, **config)+ if "verbose" not in self.params:+ self.params["verbose"] = -1+ if "regression" == task:+ from lightgbm import LGBMRegressor++ self.estimator_class = LGBMRegressor+ elif "rank" == task:+ from lightgbm import LGBMRanker++ self.estimator_class = LGBMRanker+ else:+ from lightgbm import LGBMClassifier++ self.estimator_class = LGBMClassifier+ self._time_per_iter = None+ self._train_size = 0+ self._mem_per_iter = -1+ self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0, 0) is not None++ def _preprocess(self, X):+ if (+ not isinstance(X, DataFrame)+ and issparse(X)+ and np.issubdtype(X.dtype, np.integer)+ ):+ X = X.astype(float)+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":+ # numpy array is not of numeric dtype+ X = DataFrame(X)+ for col in X.columns:+ if isinstance(X[col][0], str):+ X[col] = X[col].astype("category").cat.codes+ X = X.to_numpy()+ return X++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ start_time = time.time()+ deadline = start_time + budget if budget else np.inf+ n_iter = self.params.get(self.ITER_HP, self.DEFAULT_ITER)+ trained = False+ if not self.HAS_CALLBACK:+ mem0 = psutil.virtual_memory().available if psutil is not None else 1+ if (+ (+ not self._time_per_iter+ or abs(self._train_size - X_train.shape[0]) > 4+ )+ and budget is not None+ or self._mem_per_iter < 0+ and psutil is not None+ ) and n_iter > 1:+ self.params[self.ITER_HP] = 1+ self._t1 = self._fit(X_train, y_train, **kwargs)+ if budget is not None and self._t1 >= budget or n_iter == 1:+ return self._t1+ mem1 = psutil.virtual_memory().available if psutil is not None else 1+ self._mem1 = mem0 - mem1+ self.params[self.ITER_HP] = min(n_iter, 4)+ self._t2 = self._fit(X_train, y_train, **kwargs)+ mem2 = psutil.virtual_memory().available if psutil is not None else 1+ self._mem2 = max(mem0 - mem2, self._mem1)+ # if self._mem1 <= 0:+ # self._mem_per_iter = self._mem2 / (self.params[self.ITER_HP] + 1)+ # elif self._mem2 <= 0:+ # self._mem_per_iter = self._mem1+ # else:+ self._mem_per_iter = min(+ self._mem1, self._mem2 / self.params[self.ITER_HP]+ )+ # if self._mem_per_iter <= 1 and psutil is not None:+ # n_iter = self.params[self.ITER_HP]+ self._time_per_iter = (+ (self._t2 - self._t1) / (self.params[self.ITER_HP] - 1)+ if self._t2 > self._t1+ else self._t1+ if self._t1+ else 0.001+ )+ self._train_size = X_train.shape[0]+ if (+ budget is not None+ and self._t1 + self._t2 >= budget+ or n_iter == self.params[self.ITER_HP]+ ):+ # self.params[self.ITER_HP] = n_iter+ return time.time() - start_time+ trained = True+ # logger.debug(mem0)+ # logger.debug(self._mem_per_iter)+ if n_iter > 1:+ max_iter = min(+ n_iter,+ int(+ (budget - time.time() + start_time - self._t1)+ / self._time_per_iter+ + 1+ )+ if budget is not None+ else n_iter,+ int((1 - free_mem_ratio) * mem0 / self._mem_per_iter)+ if psutil is not None and self._mem_per_iter > 0+ else n_iter,+ )+ if trained and max_iter <= self.params[self.ITER_HP]:+ return time.time() - start_time+ # when not trained, train at least one iter+ self.params[self.ITER_HP] = max(max_iter, 1)+ if self.HAS_CALLBACK:+ kwargs_callbacks = kwargs.get("callbacks")+ if kwargs_callbacks:+ callbacks = kwargs_callbacks + self._callbacks(+ start_time, deadline, free_mem_ratio+ )+ kwargs.pop("callbacks")+ else:+ callbacks = self._callbacks(start_time, deadline, free_mem_ratio)+ if isinstance(self, XGBoostSklearnEstimator):+ from xgboost import __version__++ if __version__ >= "1.6.0":+ # since xgboost>=1.6.0, callbacks can't be passed in fit()+ self.params["callbacks"] = callbacks+ callbacks = None+ self._fit(+ X_train,+ y_train,+ callbacks=callbacks,+ **kwargs,+ )+ if callbacks is None:+ # for xgboost>=1.6.0, pop callbacks to enable pickle+ callbacks = self.params.pop("callbacks")+ self._model.set_params(callbacks=callbacks[:-1])+ best_iteration = (+ self._model.get_booster().best_iteration+ if isinstance(self, XGBoostSklearnEstimator)+ else self._model.best_iteration_+ )+ if best_iteration is not None:+ self._model.set_params(n_estimators=best_iteration + 1)+ else:+ self._fit(X_train, y_train, **kwargs)+ train_time = time.time() - start_time+ return train_time++ def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:+ return [partial(self._callback, start_time, deadline, free_mem_ratio)]++ def _callback(self, start_time, deadline, free_mem_ratio, env) -> None:+ from lightgbm.callback import EarlyStopException++ now = time.time()+ if env.iteration == 0:+ self._time_per_iter = now - start_time+ if now + self._time_per_iter > deadline:+ raise EarlyStopException(env.iteration, env.evaluation_result_list)+ if psutil is not None:+ mem = psutil.virtual_memory()+ if mem.available / mem.total < free_mem_ratio:+ raise EarlyStopException(env.iteration, env.evaluation_result_list)+++class XGBoostEstimator(SKLearnEstimator):+ """The class for tuning XGBoost regressor, not using sklearn API."""++ DEFAULT_ITER = 10++ @classmethod+ def search_space(cls, data_size, **params):+ upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower+ return {+ "n_estimators": {+ "domain": tune.lograndint(lower=4, upper=upper),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ "max_leaves": {+ "domain": tune.lograndint(lower=4, upper=upper),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ "max_depth": {+ "domain": tune.choice([0, 6, 12]),+ "init_value": 0,+ },+ "min_child_weight": {+ "domain": tune.loguniform(lower=0.001, upper=128),+ "init_value": 1.0,+ },+ "learning_rate": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1.0),+ "init_value": 0.1,+ },+ "subsample": {+ "domain": tune.uniform(lower=0.1, upper=1.0),+ "init_value": 1.0,+ },+ "colsample_bylevel": {+ "domain": tune.uniform(lower=0.01, upper=1.0),+ "init_value": 1.0,+ },+ "colsample_bytree": {+ "domain": tune.uniform(lower=0.01, upper=1.0),+ "init_value": 1.0,+ },+ "reg_alpha": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),+ "init_value": 1 / 1024,+ },+ "reg_lambda": {+ "domain": tune.loguniform(lower=1 / 1024, upper=1024),+ "init_value": 1.0,+ },+ }++ @classmethod+ def size(cls, config):+ return LGBMEstimator.size(config)++ @classmethod+ def cost_relative2lgbm(cls):+ return 1.6++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ max_depth = params["max_depth"] = params.get("max_depth", 0)+ if max_depth == 0:+ params["grow_policy"] = params.get("grow_policy", "lossguide")+ params["tree_method"] = params.get("tree_method", "hist")+ # params["booster"] = params.get("booster", "gbtree")+ params["use_label_encoder"] = params.get("use_label_encoder", False)+ if "n_jobs" in config:+ params["nthread"] = params.pop("n_jobs")+ return params++ def __init__(+ self,+ task="regression",+ **config,+ ):+ super().__init__(task, **config)+ self.params["verbosity"] = 0++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ import xgboost as xgb++ start_time = time.time()+ deadline = start_time + budget if budget else np.inf+ if issparse(X_train):+ if xgb.__version__ < "1.6.0":+ # "auto" fails for sparse input since xgboost 1.6.0+ self.params["tree_method"] = "auto"+ else:+ X_train = self._preprocess(X_train)+ if "sample_weight" in kwargs:+ dtrain = xgb.DMatrix(X_train, label=y_train, weight=kwargs["sample_weight"])+ else:+ dtrain = xgb.DMatrix(X_train, label=y_train)++ objective = self.params.get("objective")+ if isinstance(objective, str):+ obj = None+ else:+ obj = objective+ if "objective" in self.params:+ del self.params["objective"]+ _n_estimators = self.params.pop("n_estimators")+ callbacks = XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)+ if callbacks:+ self._model = xgb.train(+ self.params,+ dtrain,+ _n_estimators,+ obj=obj,+ callbacks=callbacks,+ )+ self.params["n_estimators"] = self._model.best_iteration + 1+ else:+ self._model = xgb.train(self.params, dtrain, _n_estimators, obj=obj)+ self.params["n_estimators"] = _n_estimators+ self.params["objective"] = objective+ del dtrain+ train_time = time.time() - start_time+ return train_time++ def predict(self, X, **kwargs):+ import xgboost as xgb++ if not issparse(X):+ X = self._preprocess(X)+ dtest = xgb.DMatrix(X)+ return super().predict(dtest, **kwargs)++ @classmethod+ def _callbacks(cls, start_time, deadline, free_mem_ratio):+ try:+ from xgboost.callback import TrainingCallback+ except ImportError: # for xgboost<1.3+ return None++ class ResourceLimit(TrainingCallback):+ def after_iteration(self, model, epoch, evals_log) -> bool:+ now = time.time()+ if epoch == 0:+ self._time_per_iter = now - start_time+ if now + self._time_per_iter > deadline:+ return True+ if psutil is not None:+ mem = psutil.virtual_memory()+ if mem.available / mem.total < free_mem_ratio:+ return True+ return False++ return [ResourceLimit()]+++class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):+ """The class for tuning XGBoost with unlimited depth, using sklearn API."""++ DEFAULT_ITER = 10++ @classmethod+ def search_space(cls, data_size, **params):+ space = XGBoostEstimator.search_space(data_size)+ space.pop("max_depth")+ return space++ @classmethod+ def cost_relative2lgbm(cls):+ return XGBoostEstimator.cost_relative2lgbm()++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ max_depth = params["max_depth"] = params.get("max_depth", 0)+ if max_depth == 0:+ params["grow_policy"] = params.get("grow_policy", "lossguide")+ params["tree_method"] = params.get("tree_method", "hist")+ params["use_label_encoder"] = params.get("use_label_encoder", False)+ return params++ def __init__(+ self,+ task="binary",+ **config,+ ):+ super().__init__(task, **config)+ del self.params["verbose"]+ self.params["verbosity"] = 0+ import xgboost as xgb++ self.estimator_class = xgb.XGBRegressor+ if "rank" == task:+ self.estimator_class = xgb.XGBRanker+ elif task in CLASSIFICATION:+ self.estimator_class = xgb.XGBClassifier+ self._xgb_version = xgb.__version__++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ if issparse(X_train) and self._xgb_version < "1.6.0":+ # "auto" fails for sparse input since xgboost 1.6.0+ self.params["tree_method"] = "auto"+ if kwargs.get("gpu_per_trial"):+ self.params["tree_method"] = "gpu_hist"+ kwargs.pop("gpu_per_trial")+ return super().fit(X_train, y_train, budget, free_mem_ratio, **kwargs)++ def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:+ return XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)+++class XGBoostLimitDepthEstimator(XGBoostSklearnEstimator):+ """The class for tuning XGBoost with limited depth, using sklearn API."""++ @classmethod+ def search_space(cls, data_size, **params):+ space = XGBoostEstimator.search_space(data_size)+ space.pop("max_leaves")+ upper = max(6, int(np.log2(data_size[0])))+ space["max_depth"] = {+ "domain": tune.randint(lower=1, upper=min(upper, 16)),+ "init_value": 6,+ "low_cost_init_value": 1,+ }+ space["learning_rate"]["init_value"] = 0.3+ space["n_estimators"]["init_value"] = 10+ return space++ @classmethod+ def cost_relative2lgbm(cls):+ return 64+++class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):+ """The class for tuning Random Forest."""++ HAS_CALLBACK = False+ nrows = 101++ @classmethod+ def search_space(cls, data_size, task, **params):+ RandomForestEstimator.nrows = int(data_size[0])+ upper = min(2048, RandomForestEstimator.nrows)+ init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1+ lower = min(0.1, init)+ space = {+ "n_estimators": {+ "domain": tune.lograndint(lower=4, upper=max(5, upper)),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ "max_features": {+ "domain": tune.loguniform(lower=lower, upper=1.0),+ "init_value": init,+ },+ "max_leaves": {+ "domain": tune.lograndint(+ lower=4,+ upper=max(5, min(32768, RandomForestEstimator.nrows >> 1)), #+ ),+ "init_value": 4,+ "low_cost_init_value": 4,+ },+ }+ if task in CLASSIFICATION:+ space["criterion"] = {+ "domain": tune.choice(["gini", "entropy"]),+ # "init_value": "gini",+ }+ return space++ @classmethod+ def cost_relative2lgbm(cls):+ return 2++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ if "max_leaves" in params:+ params["max_leaf_nodes"] = params.get(+ "max_leaf_nodes", params.pop("max_leaves")+ )+ if self._task not in CLASSIFICATION and "criterion" in config:+ params.pop("criterion")+ if "random_state" not in params:+ params["random_state"] = 12032022+ return params++ def __init__(+ self,+ task="binary",+ **params,+ ):+ super().__init__(task, **params)+ self.params["verbose"] = 0+ self.estimator_class = RandomForestRegressor+ if task in CLASSIFICATION:+ self.estimator_class = RandomForestClassifier+++class ExtraTreesEstimator(RandomForestEstimator):+ """The class for tuning Extra Trees."""++ @classmethod+ def cost_relative2lgbm(cls):+ return 1.9++ def __init__(self, task="binary", **params):+ super().__init__(task, **params)+ if "regression" in task:+ self.estimator_class = ExtraTreesRegressor+ else:+ self.estimator_class = ExtraTreesClassifier+++class LRL1Classifier(SKLearnEstimator):+ """The class for tuning Logistic Regression with L1 regularization."""++ @classmethod+ def search_space(cls, **params):+ return {+ "C": {+ "domain": tune.loguniform(lower=0.03125, upper=32768.0),+ "init_value": 1.0,+ },+ }++ @classmethod+ def cost_relative2lgbm(cls):+ return 160++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ params["tol"] = params.get("tol", 0.0001)+ params["solver"] = params.get("solver", "saga")+ params["penalty"] = params.get("penalty", "l1")+ return params++ def __init__(self, task="binary", **config):+ super().__init__(task, **config)+ assert task in CLASSIFICATION, "LogisticRegression for classification task only"+ self.estimator_class = LogisticRegression+++class LRL2Classifier(SKLearnEstimator):+ """The class for tuning Logistic Regression with L2 regularization."""++ limit_resource = True++ @classmethod+ def search_space(cls, **params):+ return LRL1Classifier.search_space(**params)++ @classmethod+ def cost_relative2lgbm(cls):+ return 25++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ params["tol"] = params.get("tol", 0.0001)+ params["solver"] = params.get("solver", "lbfgs")+ params["penalty"] = params.get("penalty", "l2")+ return params++ def __init__(self, task="binary", **config):+ super().__init__(task, **config)+ assert task in CLASSIFICATION, "LogisticRegression for classification task only"+ self.estimator_class = LogisticRegression+++class CatBoostEstimator(BaseEstimator):+ """The class for tuning CatBoost."""++ ITER_HP = "n_estimators"+ DEFAULT_ITER = 1000++ @classmethod+ def search_space(cls, data_size, **params):+ upper = max(min(round(1500000 / data_size[0]), 150), 12)+ return {+ "early_stopping_rounds": {+ "domain": tune.lograndint(lower=10, upper=upper),+ "init_value": 10,+ "low_cost_init_value": 10,+ },+ "learning_rate": {+ "domain": tune.loguniform(lower=0.005, upper=0.2),+ "init_value": 0.1,+ },+ "n_estimators": {+ "domain": 8192,+ "init_value": 8192,+ },+ }++ @classmethod+ def size(cls, config):+ n_estimators = config.get("n_estimators", 8192)+ max_leaves = 64+ return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8++ @classmethod+ def cost_relative2lgbm(cls):+ return 15++ def _preprocess(self, X):+ if isinstance(X, DataFrame):+ cat_columns = X.select_dtypes(include=["category"]).columns+ if not cat_columns.empty:+ X = X.copy()+ X[cat_columns] = X[cat_columns].apply(+ lambda x: x.cat.rename_categories(+ [+ str(c) if isinstance(c, float) else c+ for c in x.cat.categories+ ]+ )+ )+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":+ # numpy array is not of numeric dtype+ X = DataFrame(X)+ for col in X.columns:+ if isinstance(X[col][0], str):+ X[col] = X[col].astype("category").cat.codes+ X = X.to_numpy()+ return X++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ params["n_estimators"] = params.get("n_estimators", 8192)+ if "n_jobs" in params:+ params["thread_count"] = params.pop("n_jobs")+ return params++ def __init__(+ self,+ task="binary",+ **config,+ ):+ super().__init__(task, **config)+ self.params.update(+ {+ "verbose": config.get("verbose", False),+ "random_seed": config.get("random_seed", 10242048),+ }+ )+ from catboost import CatBoostRegressor++ self.estimator_class = CatBoostRegressor+ if task in CLASSIFICATION:+ from catboost import CatBoostClassifier++ self.estimator_class = CatBoostClassifier++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ start_time = time.time()+ deadline = start_time + budget if budget else np.inf+ train_dir = f"catboost_{str(start_time)}"+ X_train = self._preprocess(X_train)+ if isinstance(X_train, DataFrame):+ cat_features = list(X_train.select_dtypes(include="category").columns)+ else:+ cat_features = []+ use_best_model = kwargs.get("use_best_model", True)+ n = (+ max(int(len(y_train) * 0.9), len(y_train) - 1000)+ if use_best_model+ else len(y_train)+ )+ X_tr, y_tr = X_train[:n], y_train[:n]+ from catboost import Pool, __version__++ eval_set = (+ Pool(data=X_train[n:], label=y_train[n:], cat_features=cat_features)+ if use_best_model+ else None+ )+ if "sample_weight" in kwargs:+ weight = kwargs["sample_weight"]+ if weight is not None:+ kwargs["sample_weight"] = weight[:n]+ else:+ weight = None++ model = self.estimator_class(train_dir=train_dir, **self.params)+ if __version__ >= "0.26":+ model.fit(+ X_tr,+ y_tr,+ cat_features=cat_features,+ eval_set=eval_set,+ callbacks=CatBoostEstimator._callbacks(+ start_time, deadline, free_mem_ratio if use_best_model else None+ ),+ **kwargs,+ )+ else:+ model.fit(+ X_tr,+ y_tr,+ cat_features=cat_features,+ eval_set=eval_set,+ **kwargs,+ )+ shutil.rmtree(train_dir, ignore_errors=True)+ if weight is not None:+ kwargs["sample_weight"] = weight+ self._model = model+ self.params[self.ITER_HP] = self._model.tree_count_+ train_time = time.time() - start_time+ return train_time++ @classmethod+ def _callbacks(cls, start_time, deadline, free_mem_ratio):+ class ResourceLimit:+ def after_iteration(self, info) -> bool:+ now = time.time()+ if info.iteration == 1:+ self._time_per_iter = now - start_time+ if now + self._time_per_iter > deadline:+ return False+ if psutil is not None and free_mem_ratio is not None:+ mem = psutil.virtual_memory()+ if mem.available / mem.total < free_mem_ratio:+ return False+ return True # can continue++ return [ResourceLimit()]+++class KNeighborsEstimator(BaseEstimator):+ @classmethod+ def search_space(cls, data_size, **params):+ upper = min(512, int(data_size[0] / 2))+ return {+ "n_neighbors": {+ "domain": tune.lograndint(lower=1, upper=max(2, upper)),+ "init_value": 5,+ "low_cost_init_value": 1,+ },+ }++ @classmethod+ def cost_relative2lgbm(cls):+ return 30++ def config2params(self, config: dict) -> dict:+ params = super().config2params(config)+ params["weights"] = params.get("weights", "distance")+ return params++ def __init__(self, task="binary", **config):+ super().__init__(task, **config)+ if task in CLASSIFICATION:+ from sklearn.neighbors import KNeighborsClassifier++ self.estimator_class = KNeighborsClassifier+ else:+ from sklearn.neighbors import KNeighborsRegressor++ self.estimator_class = KNeighborsRegressor++ def _preprocess(self, X):+ if isinstance(X, DataFrame):+ cat_columns = X.select_dtypes(["category"]).columns+ if X.shape[1] == len(cat_columns):+ raise ValueError("kneighbor requires at least one numeric feature")+ X = X.drop(cat_columns, axis=1)+ elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":+ # drop categocial columns if any+ X = DataFrame(X)+ cat_columns = []+ for col in X.columns:+ if isinstance(X[col][0], str):+ cat_columns.append(col)+ X = X.drop(cat_columns, axis=1)+ X = X.to_numpy()+ return X+++class Prophet(SKLearnEstimator):+ """The class for tuning Prophet."""++ @classmethod+ def search_space(cls, **params):+ space = {+ "changepoint_prior_scale": {+ "domain": tune.loguniform(lower=0.001, upper=0.05),+ "init_value": 0.05,+ "low_cost_init_value": 0.001,+ },+ "seasonality_prior_scale": {+ "domain": tune.loguniform(lower=0.01, upper=10),+ "init_value": 10,+ },+ "holidays_prior_scale": {+ "domain": tune.loguniform(lower=0.01, upper=10),+ "init_value": 10,+ },+ "seasonality_mode": {+ "domain": tune.choice(["additive", "multiplicative"]),+ "init_value": "multiplicative",+ },+ }+ return space++ def __init__(self, task="ts_forecast", n_jobs=1, **params):+ super().__init__(task, **params)++ def _join(self, X_train, y_train):+ assert TS_TIMESTAMP_COL in X_train, (+ "Dataframe for training ts_forecast model must have column"+ f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'+ )+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])+ train_df = X_train.join(y_train)+ return train_df++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ from prophet import Prophet++ current_time = time.time()+ train_df = self._join(X_train, y_train)+ train_df = self._preprocess(train_df)+ cols = list(train_df)+ cols.remove(TS_TIMESTAMP_COL)+ cols.remove(TS_VALUE_COL)+ logging.getLogger("prophet").setLevel(logging.WARNING)+ model = Prophet(**self.params)+ for regressor in cols:+ model.add_regressor(regressor)+ with suppress_stdout_stderr():+ model.fit(train_df)+ train_time = time.time() - current_time+ self._model = model+ return train_time++ def predict(self, X, **kwargs):+ if isinstance(X, int):+ raise ValueError(+ "predict() with steps is only supported for arima/sarimax."+ " For Prophet, pass a dataframe with the first column containing"+ " the timestamp values."+ )+ if self._model is not None:+ X = self._preprocess(X)+ forecast = self._model.predict(X, **kwargs)+ return forecast["yhat"]+ else:+ logger.warning(+ "Estimator is not fit yet. Please run fit() before predict()."+ )+ return np.ones(X.shape[0])++ def score(self, X_val: DataFrame, y_val: Series, **kwargs):+ from sklearn.metrics import r2_score+ from .ml import metric_loss_score++ y_pred = self.predict(X_val, **kwargs)+ self._metric = kwargs.get("metric", None)+ if self._metric:+ return metric_loss_score(self._metric, y_pred, y_val)+ else:+ return r2_score(y_pred, y_val)+++class ARIMA(Prophet):+ """The class for tuning ARIMA."""++ @classmethod+ def search_space(cls, **params):+ space = {+ "p": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 2,+ "low_cost_init_value": 0,+ },+ "d": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 2,+ "low_cost_init_value": 0,+ },+ "q": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ }+ return space++ def _join(self, X_train, y_train):+ train_df = super()._join(X_train, y_train)+ train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])+ train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)+ return train_df++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ import warnings++ warnings.filterwarnings("ignore")+ from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator++ current_time = time.time()+ train_df = self._join(X_train, y_train)+ train_df = self._preprocess(train_df)+ regressors = list(train_df)+ regressors.remove(TS_VALUE_COL)+ if regressors:+ model = ARIMA_estimator(+ train_df[[TS_VALUE_COL]],+ exog=train_df[regressors],+ order=(self.params["p"], self.params["d"], self.params["q"]),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ else:+ model = ARIMA_estimator(+ train_df,+ order=(self.params["p"], self.params["d"], self.params["q"]),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ with suppress_stdout_stderr():+ model = model.fit()+ train_time = time.time() - current_time+ self._model = model+ return train_time++ def predict(self, X, **kwargs):+ if self._model is not None:+ if isinstance(X, int):+ forecast = self._model.forecast(steps=X)+ elif isinstance(X, DataFrame):+ start = X[TS_TIMESTAMP_COL].iloc[0]+ end = X[TS_TIMESTAMP_COL].iloc[-1]+ if len(X.columns) > 1:+ X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))+ regressors = list(X)+ forecast = self._model.predict(+ start=start, end=end, exog=X[regressors], **kwargs+ )+ else:+ forecast = self._model.predict(start=start, end=end, **kwargs)+ else:+ raise ValueError(+ "X needs to be either a pandas Dataframe with dates as the first column"+ " or an int number of periods for predict()."+ )+ return forecast+ else:+ return np.ones(X if isinstance(X, int) else X.shape[0])+++class SARIMAX(ARIMA):+ """The class for tuning SARIMA."""++ @classmethod+ def search_space(cls, **params):+ space = {+ "p": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 2,+ "low_cost_init_value": 0,+ },+ "d": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 2,+ "low_cost_init_value": 0,+ },+ "q": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "P": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "D": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "Q": {+ "domain": tune.qrandint(lower=0, upper=10, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "s": {+ "domain": tune.choice([1, 4, 6, 12]),+ "init_value": 12,+ },+ }+ return space++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ import warnings++ warnings.filterwarnings("ignore")+ from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator++ current_time = time.time()+ train_df = self._join(X_train, y_train)+ train_df = self._preprocess(train_df)+ regressors = list(train_df)+ regressors.remove(TS_VALUE_COL)+ if regressors:+ model = SARIMAX_estimator(+ train_df[[TS_VALUE_COL]],+ exog=train_df[regressors],+ order=(self.params["p"], self.params["d"], self.params["q"]),+ seasonal_order=(+ self.params["P"],+ self.params["D"],+ self.params["Q"],+ self.params["s"],+ ),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ else:+ model = SARIMAX_estimator(+ train_df,+ order=(self.params["p"], self.params["d"], self.params["q"]),+ seasonal_order=(+ self.params["P"],+ self.params["D"],+ self.params["Q"],+ self.params["s"],+ ),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ with suppress_stdout_stderr():+ model = model.fit()+ train_time = time.time() - current_time+ self._model = model+ return train_time+++class TS_SKLearn(SKLearnEstimator):+ """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""++ base_class = SKLearnEstimator++ @classmethod+ def search_space(cls, data_size, pred_horizon, **params):+ space = cls.base_class.search_space(data_size, **params)+ space.update(+ {+ "optimize_for_horizon": {+ "domain": tune.choice([True, False]),+ "init_value": False,+ "low_cost_init_value": False,+ },+ "lags": {+ "domain": tune.randint(+ lower=1, upper=max(2, int(np.sqrt(data_size[0])))+ ),+ "init_value": 3,+ },+ }+ )+ return space++ def __init__(self, task="ts_forecast", **params):+ super().__init__(task, **params)+ self.hcrystaball_model = None+ self.ts_task = (+ "regression" if task in TS_FORECASTREGRESSION else "classification"+ )++ def transform_X(self, X):+ cols = list(X)+ if len(cols) == 1:+ ds_col = cols[0]+ X = DataFrame(index=X[ds_col])+ elif len(cols) > 1:+ ds_col = cols[0]+ exog_cols = cols[1:]+ X = X[exog_cols].set_index(X[ds_col])+ return X++ def _fit(self, X_train, y_train, budget=None, **kwargs):+ from hcrystalball.wrappers import get_sklearn_wrapper++ X_train = self.transform_X(X_train)+ X_train = self._preprocess(X_train)+ params = self.params.copy()+ lags = params.pop("lags")+ optimize_for_horizon = params.pop("optimize_for_horizon")+ estimator = self.base_class(task=self.ts_task, **params)+ self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)+ self.hcrystaball_model.lags = int(lags)+ self.hcrystaball_model.fit(X_train, y_train)+ if optimize_for_horizon:+ # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon+ model_list = []+ for i in range(1, kwargs["period"] + 1):+ (+ X_fit,+ y_fit,+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(+ X_train, y_train, i+ )+ self.hcrystaball_model.model.set_params(**estimator.params)+ model = self.hcrystaball_model.model.fit(X_fit, y_fit)+ model_list.append(model)+ self._model = model_list+ else:+ (+ X_fit,+ y_fit,+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(+ X_train, y_train, kwargs["period"]+ )+ self.hcrystaball_model.model.set_params(**estimator.params)+ model = self.hcrystaball_model.model.fit(X_fit, y_fit)+ self._model = model++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ current_time = time.time()+ self._fit(X_train, y_train, budget=budget, **kwargs)+ train_time = time.time() - current_time+ return train_time++ def predict(self, X, **kwargs):+ if self._model is not None:+ X = self.transform_X(X)+ X = self._preprocess(X)+ if isinstance(self._model, list):+ assert len(self._model) == len(+ X+ ), "Model is optimized for horizon, length of X must be equal to `period`."+ preds = []+ for i in range(1, len(self._model) + 1):+ (+ X_pred,+ _,+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(+ X.iloc[:i, :]+ )+ preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])+ forecast = DataFrame(+ data=np.asarray(preds).reshape(-1, 1),+ columns=[self.hcrystaball_model.name],+ index=X.index,+ )+ else:+ (+ X_pred,+ _,+ ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)+ forecast = self._model.predict(X_pred, **kwargs)+ return forecast+ else:+ logger.warning(+ "Estimator is not fit yet. Please run fit() before predict()."+ )+ return np.ones(X.shape[0])+++class LGBM_TS(TS_SKLearn):+ """The class for tuning LGBM Regressor for time-series forecasting"""++ base_class = LGBMEstimator+++class XGBoost_TS(TS_SKLearn):+ """The class for tuning XGBoost Regressor for time-series forecasting"""++ base_class = XGBoostSklearnEstimator+++# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball+# class CatBoost_TS_Regressor(TS_Regressor):+# base_class = CatBoostEstimator+++class RF_TS(TS_SKLearn):+ """The class for tuning Random Forest Regressor for time-series forecasting"""++ base_class = RandomForestEstimator+++class ExtraTrees_TS(TS_SKLearn):+ """The class for tuning Extra Trees Regressor for time-series forecasting"""++ base_class = ExtraTreesEstimator+++class XGBoostLimitDepth_TS(TS_SKLearn):+ """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""++ base_class = XGBoostLimitDepthEstimator+++class TemporalFusionTransformerEstimator(SKLearnEstimator):+ """The class for tuning Temporal Fusion Transformer"""++ @classmethod+ def search_space(cls, data_size, pred_horizon, **params):+ space = {+ "gradient_clip_val": {+ "domain": tune.loguniform(lower=0.01, upper=100.0),+ "init_value": 0.01,+ },+ "hidden_size": {+ "domain": tune.lograndint(lower=8, upper=512),+ "init_value": 16,+ },+ "hidden_continuous_size": {+ "domain": tune.randint(lower=1, upper=65),+ "init_value": 8,+ },+ "attention_head_size": {+ "domain": tune.randint(lower=1, upper=5),+ "init_value": 4,+ },+ "dropout": {+ "domain": tune.uniform(lower=0.1, upper=0.3),+ "init_value": 0.1,+ },+ "learning_rate": {+ "domain": tune.loguniform(lower=0.00001, upper=1.0),+ "init_value": 0.001,+ },+ }+ return space++ def transform_ds(self, X_train, y_train, **kwargs):+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])+ self.data = X_train.join(y_train)++ max_prediction_length = kwargs["period"]+ self.max_encoder_length = kwargs["max_encoder_length"]+ training_cutoff = self.data["time_idx"].max() - max_prediction_length++ from pytorch_forecasting import TimeSeriesDataSet+ from pytorch_forecasting.data import GroupNormalizer++ self.group_ids = kwargs["group_ids"].copy()+ training = TimeSeriesDataSet(+ self.data[lambda x: x.time_idx <= training_cutoff],+ time_idx="time_idx",+ target=TS_VALUE_COL,+ group_ids=self.group_ids,+ min_encoder_length=kwargs.get(+ "min_encoder_length", self.max_encoder_length // 2+ ), # keep encoder length long (as it is in the validation set)+ max_encoder_length=self.max_encoder_length,+ min_prediction_length=1,+ max_prediction_length=max_prediction_length,+ static_categoricals=kwargs.get("static_categoricals", []),+ static_reals=kwargs.get("static_reals", []),+ time_varying_known_categoricals=kwargs.get(+ "time_varying_known_categoricals", []+ ),+ time_varying_known_reals=kwargs.get("time_varying_known_reals", []),+ time_varying_unknown_categoricals=kwargs.get(+ "time_varying_unknown_categoricals", []+ ),+ time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),+ variable_groups=kwargs.get(+ "variable_groups", {}+ ), # group of categorical variables can be treated as one variable+ lags=kwargs.get("lags", {}),+ target_normalizer=GroupNormalizer(+ groups=kwargs["group_ids"], transformation="softplus"+ ), # use softplus and normalize by group+ add_relative_time_idx=True,+ add_target_scales=True,+ add_encoder_length=True,+ )++ # create validation set (predict=True) which means to predict the last max_prediction_length points in time+ # for each series+ validation = TimeSeriesDataSet.from_dataset(+ training, self.data, predict=True, stop_randomization=True+ )++ # create dataloaders for model+ batch_size = kwargs.get("batch_size", 64)+ train_dataloader = training.to_dataloader(+ train=True, batch_size=batch_size, num_workers=0+ )+ val_dataloader = validation.to_dataloader(+ train=False, batch_size=batch_size * 10, num_workers=0+ )++ return training, train_dataloader, val_dataloader++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ import warnings+ import pytorch_lightning as pl+ from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor+ import torch+ from pytorch_forecasting import TemporalFusionTransformer+ from pytorch_forecasting.metrics import QuantileLoss++ warnings.filterwarnings("ignore")+ current_time = time.time()+ training, train_dataloader, val_dataloader = self.transform_ds(+ X_train, y_train, **kwargs+ )+ params = self.params.copy()+ gradient_clip_val = params.pop("gradient_clip_val")+ params.pop("n_jobs")+ max_epochs = kwargs.get("max_epochs", 20)+ early_stop_callback = EarlyStopping(+ monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min"+ )++ def _fit(log):+ default_trainer_kwargs = dict(+ gpus=kwargs.get("gpu_per_trial", [0])+ if torch.cuda.is_available()+ else None,+ max_epochs=max_epochs,+ gradient_clip_val=gradient_clip_val,+ callbacks=[LearningRateMonitor(), early_stop_callback]+ if log+ else [early_stop_callback],+ logger=log,+ )+ trainer = pl.Trainer(+ **default_trainer_kwargs,+ )+ tft = TemporalFusionTransformer.from_dataset(+ training,+ **params,+ lstm_layers=2, # 2 is mostly optimal according to documentation+ output_size=7, # 7 quantiles by default+ loss=QuantileLoss(),+ log_interval=10 if log else 0,+ # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches+ reduce_on_plateau_patience=4,+ )+ # fit network+ trainer.fit(+ tft,+ train_dataloaders=train_dataloader,+ val_dataloaders=val_dataloader,+ )+ return trainer++ # try:+ # from pytorch_lightning.loggers import TensorBoardLogger++ # logger = TensorBoardLogger(+ # kwargs.get("log_dir", "lightning_logs")+ # ) # logging results to a tensorboard+ # trainer = _fit(log=logger)+ # except ValueError:+ # issue with pytorch forecasting model log_prediction() function+ # pytorch-forecasting issue #1145+ trainer = _fit(log=False)+ best_model_path = trainer.checkpoint_callback.best_model_path+ best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)+ train_time = time.time() - current_time+ self._model = best_tft+ return train_time++ def predict(self, X):+ import pandas as pd++ ids = self.group_ids.copy()+ ids.append(TS_TIMESTAMP_COL)+ encoder_data = self.data[+ lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length+ ]+ # following pytorchforecasting example, make all target values equal to the last data+ last_data_cols = self.group_ids.copy()+ last_data_cols.append(TS_VALUE_COL)+ last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]+ decoder_data = X+ if "time_idx" not in decoder_data:+ decoder_data = add_time_idx_col(decoder_data)+ decoder_data["time_idx"] += (+ encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()+ )+ # decoder_data[TS_VALUE_COL] = 0+ decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)+ decoder_data = decoder_data.sort_values(ids)+ new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)+ new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")+ new_raw_predictions = self._model.predict(new_prediction_data)+ index = [decoder_data[idx].to_numpy() for idx in ids]+ predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)+ return predictions+++class suppress_stdout_stderr(object):+ def __init__(self):+ # Open a pair of null files+ self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]+ # Save the actual stdout (1) and stderr (2) file descriptors.+ self.save_fds = (os.dup(1), os.dup(2))++ def __enter__(self):+ # Assign the null pointers to stdout and stderr.+ os.dup2(self.null_fds[0], 1)+ os.dup2(self.null_fds[1], 2)++ def __exit__(self, *_):+ # Re-assign the real stdout/stderr back to (1) and (2)+ os.dup2(self.save_fds[0], 1)+ os.dup2(self.save_fds[1], 2)+ # Close the null files+ os.close(self.null_fds[0])+ os.close(self.null_fds[1])diff --git a/flaml/nlp/README.md b/flaml/automl/nlp/README.mdsimilarity index 100%rename from flaml/nlp/README.mdrename to flaml/automl/nlp/README.mddiff --git a/flaml/nlp/__init__.py b/flaml/automl/nlp/__init__.pysimilarity index 100%rename from flaml/nlp/__init__.pyrename to flaml/automl/nlp/__init__.pydiff --git a/flaml/nlp/huggingface/__init__.py b/flaml/automl/nlp/huggingface/__init__.pysimilarity index 100%rename from flaml/nlp/huggingface/__init__.pyrename to flaml/automl/nlp/huggingface/__init__.pydiff --git a/flaml/nlp/huggingface/data_collator.py b/flaml/automl/nlp/huggingface/data_collator.pysimilarity index 98%rename from flaml/nlp/huggingface/data_collator.pyrename to flaml/automl/nlp/huggingface/data_collator.pyindex 330846198..484e6a346 100644--- a/flaml/nlp/huggingface/data_collator.py+++ b/flaml/automl/nlp/huggingface/data_collator.py@@ -6,7 +6,7 @@ from transformers.data.data_collator import ()from collections import OrderedDict-from flaml.data import (+from flaml.automl.data import (TOKENCLASSIFICATION,MULTICHOICECLASSIFICATION,SUMMARIZATION,diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/automl/nlp/huggingface/trainer.pysimilarity index 100%rename from flaml/nlp/huggingface/trainer.pyrename to flaml/automl/nlp/huggingface/trainer.pydiff --git a/flaml/nlp/huggingface/training_args.py b/flaml/automl/nlp/huggingface/training_args.pysimilarity index 98%rename from flaml/nlp/huggingface/training_args.pyrename to flaml/automl/nlp/huggingface/training_args.pyindex c3eb839b6..b064f51f3 100644--- a/flaml/nlp/huggingface/training_args.py+++ b/flaml/automl/nlp/huggingface/training_args.py@@ -1,7 +1,7 @@import argparsefrom dataclasses import dataclass, field-from ...data import (+from flaml.automl.data import (NLG_TASKS,)from typing import Optional, List@@ -33,7 +33,7 @@ class TrainingArgumentsForAuto(TrainingArguments):When the task is sequence labeling/token classification, there are two formats of the labels:(1) The token labels, i.e., [B-PER, I-PER, B-LOC]; (2) Id labels. For (2), need to pass the label_list (e.g., [B-PER, I-PER, B-LOC])to convert the Id to token labels when computing the metric with metric_loss_score.- See the example in [a simple token classification example](../../../Examples/AutoML-NLP#a-simple-token-classification-example).+ See the example in [a simple token classification example](../../../../Examples/AutoML-NLP#a-simple-token-classification-example)."""task: str = field(default="seq-classification")diff --git a/flaml/nlp/huggingface/utils.py b/flaml/automl/nlp/huggingface/utils.pysimilarity index 99%rename from flaml/nlp/huggingface/utils.pyrename to flaml/automl/nlp/huggingface/utils.pyindex 13f1931fd..ba7558f09 100644--- a/flaml/nlp/huggingface/utils.py+++ b/flaml/automl/nlp/huggingface/utils.py@@ -2,7 +2,7 @@ import pandas as pdfrom itertools import chainimport numpy as np-from ...data import (+from flaml.automl.data import (SUMMARIZATION,SEQREGRESSION,SEQCLASSIFICATION,diff --git a/flaml/nlp/utils.py b/flaml/automl/nlp/utils.pysimilarity index 98%rename from flaml/nlp/utils.pyrename to flaml/automl/nlp/utils.pyindex 5e69bead0..872c1d037 100644--- a/flaml/nlp/utils.py+++ b/flaml/automl/nlp/utils.py@@ -1,13 +1,12 @@from typing import Dict, Anyimport numpy as np-from ..data import (+from flaml.automl.data import (SUMMARIZATION,SEQREGRESSION,SEQCLASSIFICATION,MULTICHOICECLASSIFICATION,TOKENCLASSIFICATION,- NLG_TASKS,)diff --git a/flaml/training_log.py b/flaml/automl/training_log.pysimilarity index 100%rename from flaml/training_log.pyrename to flaml/automl/training_log.pydiff --git a/flaml/data.py b/flaml/data.pyindex 9deab1b79..522b47fe0 100644--- a/flaml/data.py+++ b/flaml/data.py@@ -1,485 +1,9 @@-# !-# * Copyright (c) Microsoft Corporation. All rights reserved.-# * Licensed under the MIT License. See LICENSE file in the-# * project root for license information.-import numpy as np-from scipy.sparse import vstack, issparse-import pandas as pd-from pandas import DataFrame, Series+import warnings-from .training_log import training_log_reader+from flaml.automl.data import *-from datetime import datetime-from typing import Dict, Union, List-# TODO: if your task is not specified in here, define your task as an all-capitalized word-SEQCLASSIFICATION = "seq-classification"-MULTICHOICECLASSIFICATION = "multichoice-classification"-TOKENCLASSIFICATION = "token-classification"-CLASSIFICATION = (- "binary",- "multiclass",- "classification",- SEQCLASSIFICATION,- MULTICHOICECLASSIFICATION,- TOKENCLASSIFICATION,+warnings.warn(+ "Importing from `flaml.data` is deprecated. Please use `flaml.automl.data`.",+ DeprecationWarning,)-SEQREGRESSION = "seq-regression"-REGRESSION = ("regression", SEQREGRESSION)-TS_FORECASTREGRESSION = (- "forecast",- "ts_forecast",- "ts_forecast_regression",-)-TS_FORECASTCLASSIFICATION = "ts_forecast_classification"-TS_FORECASTPANEL = "ts_forecast_panel"-TS_FORECAST = (- *TS_FORECASTREGRESSION,- TS_FORECASTCLASSIFICATION,- TS_FORECASTPANEL,-)-TS_TIMESTAMP_COL = "ds"-TS_VALUE_COL = "y"-SUMMARIZATION = "summarization"-NLG_TASKS = (SUMMARIZATION,)-NLU_TASKS = (- SEQREGRESSION,- SEQCLASSIFICATION,- MULTICHOICECLASSIFICATION,- TOKENCLASSIFICATION,-)---def _is_nlp_task(task):- if task in NLU_TASKS or task in NLG_TASKS:- return True- else:- return False---def load_openml_dataset(- dataset_id, data_dir=None, random_state=0, dataset_format="dataframe"-):- """Load dataset from open ML.-- If the file is not cached locally, download it from open ML.-- Args:- dataset_id: An integer of the dataset id in openml.- data_dir: A string of the path to store and load the data.- random_state: An integer of the random seed for splitting data.- dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'.- Can choose from ['dataframe', 'array'].- If 'dataframe', the returned dataset will be a Pandas DataFrame.- If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix.-- Returns:- X_train: Training data.- X_test: Test data.- y_train: A series or array of labels for training data.- y_test: A series or array of labels for test data.- """- import os- import openml- import pickle- from sklearn.model_selection import train_test_split-- filename = "openml_ds" + str(dataset_id) + ".pkl"- filepath = os.path.join(data_dir, filename)- if os.path.isfile(filepath):- print("load dataset from", filepath)- with open(filepath, "rb") as f:- dataset = pickle.load(f)- else:- print("download dataset from openml")- dataset = openml.datasets.get_dataset(dataset_id)- if not os.path.exists(data_dir):- os.makedirs(data_dir)- with open(filepath, "wb") as f:- pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)- print("Dataset name:", dataset.name)- try:- X, y, *__ = dataset.get_data(- target=dataset.default_target_attribute, dataset_format=dataset_format- )- except ValueError:- from sklearn.datasets import fetch_openml-- X, y = fetch_openml(data_id=dataset_id, return_X_y=True)- X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state)- print(- "X_train.shape: {}, y_train.shape: {};\nX_test.shape: {}, y_test.shape: {}".format(- X_train.shape,- y_train.shape,- X_test.shape,- y_test.shape,- )- )- return X_train, X_test, y_train, y_test---def load_openml_task(task_id, data_dir):- """Load task from open ML.-- Use the first fold of the task.- If the file is not cached locally, download it from open ML.-- Args:- task_id: An integer of the task id in openml.- data_dir: A string of the path to store and load the data.-- Returns:- X_train: A dataframe of training data.- X_test: A dataframe of test data.- y_train: A series of labels for training data.- y_test: A series of labels for test data.- """- import os- import openml- import pickle-- task = openml.tasks.get_task(task_id)- filename = "openml_task" + str(task_id) + ".pkl"- filepath = os.path.join(data_dir, filename)- if os.path.isfile(filepath):- print("load dataset from", filepath)- with open(filepath, "rb") as f:- dataset = pickle.load(f)- else:- print("download dataset from openml")- dataset = task.get_dataset()- with open(filepath, "wb") as f:- pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)- X, y, _, _ = dataset.get_data(task.target_name)- train_indices, test_indices = task.get_train_test_split_indices(- repeat=0,- fold=0,- sample=0,- )- X_train = X.iloc[train_indices]- y_train = y[train_indices]- X_test = X.iloc[test_indices]- y_test = y[test_indices]- print(- "X_train.shape: {}, y_train.shape: {},\nX_test.shape: {}, y_test.shape: {}".format(- X_train.shape,- y_train.shape,- X_test.shape,- y_test.shape,- )- )- return X_train, X_test, y_train, y_test---def get_output_from_log(filename, time_budget):- """Get output from log file.-- Args:- filename: A string of the log file name.- time_budget: A float of the time budget in seconds.-- Returns:- search_time_list: A list of the finished time of each logged iter.- best_error_list: A list of the best validation error after each logged iter.- error_list: A list of the validation error of each logged iter.- config_list: A list of the estimator, sample size and config of each logged iter.- logged_metric_list: A list of the logged metric of each logged iter.- """-- best_config = None- best_learner = None- best_val_loss = float("+inf")-- search_time_list = []- config_list = []- best_error_list = []- error_list = []- logged_metric_list = []- best_config_list = []- with training_log_reader(filename) as reader:- for record in reader.records():- time_used = record.wall_clock_time- val_loss = record.validation_loss- config = record.config- learner = record.learner.split("_")[0]- sample_size = record.sample_size- metric = record.logged_metric-- if time_used < time_budget and np.isfinite(val_loss):- if val_loss < best_val_loss:- best_val_loss = val_loss- best_config = config- best_learner = learner- best_config_list.append(best_config)- search_time_list.append(time_used)- best_error_list.append(best_val_loss)- logged_metric_list.append(metric)- error_list.append(val_loss)- config_list.append(- {- "Current Learner": learner,- "Current Sample": sample_size,- "Current Hyper-parameters": record.config,- "Best Learner": best_learner,- "Best Hyper-parameters": best_config,- }- )-- return (- search_time_list,- best_error_list,- error_list,- config_list,- logged_metric_list,- )---def concat(X1, X2):- """concatenate two matrices vertically."""- if isinstance(X1, (DataFrame, Series)):- df = pd.concat([X1, X2], sort=False)- df.reset_index(drop=True, inplace=True)- if isinstance(X1, DataFrame):- cat_columns = X1.select_dtypes(include="category").columns- if len(cat_columns):- df[cat_columns] = df[cat_columns].astype("category")- return df- if issparse(X1):- return vstack((X1, X2))- else:- return np.concatenate([X1, X2])---def add_time_idx_col(X):- unique_dates = X[TS_TIMESTAMP_COL].drop_duplicates().sort_values(ascending=True)- # assume no missing timestamps- freq = pd.infer_freq(unique_dates)- if freq == "MS":- X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year * 12 + X[TS_TIMESTAMP_COL].dt.month- elif freq == "Y":- X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year- else:- # using time frequency to generate all time stamps and then indexing for time_idx- # full_range = pd.date_range(X[TS_TIMESTAMP_COL].min(), X[TS_TIMESTAMP_COL].max(), freq=freq).to_list()- # X["time_idx"] = [full_range.index(time) for time in X[TS_TIMESTAMP_COL]]- # taking minimum difference in timestamp- timestamps = unique_dates.view("int64")- freq = int(timestamps.diff().mode())- X["time_idx"] = timestamps - timestamps.min() / freq- X["time_idx"] = X["time_idx"].astype("int")- return X---class DataTransformer:- """Transform input training data."""-- def fit_transform(self, X: Union[DataFrame, np.array], y, task):- """Fit transformer and process the input training data according to the task type.-- Args:- X: A numpy array or a pandas dataframe of training data.- y: A numpy array or a pandas series of labels.- task: A string of the task type, e.g.,- 'classification', 'regression', 'ts_forecast', 'rank'.-- Returns:- X: Processed numpy array or pandas dataframe of training data.- y: Processed numpy array or pandas series of labels.- """- if _is_nlp_task(task):- # if the mode is NLP, check the type of input, each column must be either string or- # ids (input ids, token type id, attention mask, etc.)- str_columns = []- for column in X.columns:- if isinstance(X[column].iloc[0], str):- str_columns.append(column)- if len(str_columns) > 0:- X[str_columns] = X[str_columns].astype("string")- self._str_columns = str_columns- elif isinstance(X, DataFrame):- X = X.copy()- n = X.shape[0]- cat_columns, num_columns, datetime_columns = [], [], []- drop = False- if task in TS_FORECAST:- X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})- if task is TS_FORECASTPANEL:- if "time_idx" not in X:- X = add_time_idx_col(X)- ds_col = X.pop(TS_TIMESTAMP_COL)- if isinstance(y, Series):- y = y.rename(TS_VALUE_COL)- for column in X.columns:- # sklearn\utils\validation.py needs int/float values- if X[column].dtype.name in ("object", "category"):- if (- X[column].nunique() == 1- or X[column].nunique(dropna=True)- == n - X[column].isnull().sum()- ):- X.drop(columns=column, inplace=True)- drop = True- elif X[column].dtype.name == "category":- current_categories = X[column].cat.categories- if "__NAN__" not in current_categories:- X[column] = (- X[column]- .cat.add_categories("__NAN__")- .fillna("__NAN__")- )- cat_columns.append(column)- else:- X[column] = X[column].fillna("__NAN__")- cat_columns.append(column)- elif X[column].nunique(dropna=True) < 2:- X.drop(columns=column, inplace=True)- drop = True- else: # datetime or numeric- if X[column].dtype.name == "datetime64[ns]":- tmp_dt = X[column].dt- new_columns_dict = {- f"year_{column}": tmp_dt.year,- f"month_{column}": tmp_dt.month,- f"day_{column}": tmp_dt.day,- f"hour_{column}": tmp_dt.hour,- f"minute_{column}": tmp_dt.minute,- f"second_{column}": tmp_dt.second,- f"dayofweek_{column}": tmp_dt.dayofweek,- f"dayofyear_{column}": tmp_dt.dayofyear,- f"quarter_{column}": tmp_dt.quarter,- }- for key, value in new_columns_dict.items():- if (- key not in X.columns- and value.nunique(dropna=False) >= 2- ):- X[key] = value- num_columns.append(key)- X[column] = X[column].map(datetime.toordinal)- datetime_columns.append(column)- del tmp_dt- X[column] = X[column].fillna(np.nan)- num_columns.append(column)- X = X[cat_columns + num_columns]- if task in TS_FORECAST:- X.insert(0, TS_TIMESTAMP_COL, ds_col)- if cat_columns:- X[cat_columns] = X[cat_columns].astype("category")- if num_columns:- X_num = X[num_columns]- if np.issubdtype(X_num.columns.dtype, np.integer) and (- drop- or min(X_num.columns) != 0- or max(X_num.columns) != X_num.shape[1] - 1- ):- X_num.columns = range(X_num.shape[1])- drop = True- else:- drop = False- from sklearn.impute import SimpleImputer- from sklearn.compose import ColumnTransformer-- self.transformer = ColumnTransformer(- [- (- "continuous",- SimpleImputer(missing_values=np.nan, strategy="median"),- X_num.columns,- )- ]- )- X[num_columns] = self.transformer.fit_transform(X_num)- self._cat_columns, self._num_columns, self._datetime_columns = (- cat_columns,- num_columns,- datetime_columns,- )- self._drop = drop- if (- task in CLASSIFICATION- or not pd.api.types.is_numeric_dtype(y)- and task not in NLG_TASKS- ):- if task != TOKENCLASSIFICATION:- from sklearn.preprocessing import LabelEncoder-- self.label_transformer = LabelEncoder()- else:- from .nlp.utils import LabelEncoderforTokenClassification-- self.label_transformer = LabelEncoderforTokenClassification()- y = self.label_transformer.fit_transform(y)-- else:- self.label_transformer = None- self._task = task- return X, y-- def transform(self, X: Union[DataFrame, np.array]):- """Process data using fit transformer.-- Args:- X: A numpy array or a pandas dataframe of training data.-- Returns:- X: Processed numpy array or pandas dataframe of training data.- """- X = X.copy()-- if _is_nlp_task(self._task):- # if the mode is NLP, check the type of input, each column must be either string or- # ids (input ids, token type id, attention mask, etc.)- if len(self._str_columns) > 0:- X[self._str_columns] = X[self._str_columns].astype("string")- elif isinstance(X, DataFrame):- cat_columns, num_columns, datetime_columns = (- self._cat_columns,- self._num_columns,- self._datetime_columns,- )- if self._task in TS_FORECAST:- X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})- ds_col = X.pop(TS_TIMESTAMP_COL)- for column in datetime_columns:- tmp_dt = X[column].dt- new_columns_dict = {- f"year_{column}": tmp_dt.year,- f"month_{column}": tmp_dt.month,- f"day_{column}": tmp_dt.day,- f"hour_{column}": tmp_dt.hour,- f"minute_{column}": tmp_dt.minute,- f"second_{column}": tmp_dt.second,- f"dayofweek_{column}": tmp_dt.dayofweek,- f"dayofyear_{column}": tmp_dt.dayofyear,- f"quarter_{column}": tmp_dt.quarter,- }- for new_col_name, new_col_value in new_columns_dict.items():- if new_col_name not in X.columns and new_col_name in num_columns:- X[new_col_name] = new_col_value- X[column] = X[column].map(datetime.toordinal)- del tmp_dt- X = X[cat_columns + num_columns].copy()- if self._task in TS_FORECAST:- X.insert(0, TS_TIMESTAMP_COL, ds_col)- for column in cat_columns:- if X[column].dtype.name == "object":- X[column] = X[column].fillna("__NAN__")- elif X[column].dtype.name == "category":- current_categories = X[column].cat.categories- if "__NAN__" not in current_categories:- X[column] = (- X[column].cat.add_categories("__NAN__").fillna("__NAN__")- )- if cat_columns:- X[cat_columns] = X[cat_columns].astype("category")- if num_columns:- X_num = X[num_columns].fillna(np.nan)- if self._drop:- X_num.columns = range(X_num.shape[1])- X[num_columns] = self.transformer.transform(X_num)- return X---def group_counts(groups):- _, i, c = np.unique(groups, return_counts=True, return_index=True)- return c[np.argsort(i)]diff --git a/flaml/default/estimator.py b/flaml/default/estimator.pyindex ca7c980da..f5eaf5875 100644--- a/flaml/default/estimator.py+++ b/flaml/default/estimator.py@@ -1,6 +1,6 @@import sklearn.ensemble as ensemblefrom functools import wraps-from flaml.data import CLASSIFICATION+from flaml.automl.data import CLASSIFICATIONfrom .suggest import preprocess_and_suggest_hyperparamsDEFAULT_LOCATION = "default_location"diff --git a/flaml/default/suggest.py b/flaml/default/suggest.pyindex fbb02b069..71f4d74d8 100644--- a/flaml/default/suggest.py+++ b/flaml/default/suggest.py@@ -3,8 +3,8 @@ from sklearn.neighbors import NearestNeighborsimport loggingimport pathlibimport json-from flaml.data import CLASSIFICATION, DataTransformer-from flaml.ml import get_estimator_class, get_classification_objective+from flaml.automl.data import CLASSIFICATION, DataTransformer+from flaml.automl.ml import get_estimator_class, get_classification_objectivefrom flaml.version import __version__LOCATION = pathlib.Path(__file__).parent.resolve()diff --git a/flaml/ml.py b/flaml/ml.pyindex ea7edfbd5..fcc3eb98a 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -1,749 +1,9 @@-# !-# * Copyright (c) FLAML authors. All rights reserved.-# * Licensed under the MIT License. See LICENSE file in the-# * project root for license information.-import time-import numpy as np-import pandas as pd-from sklearn.metrics import (- mean_squared_error,- r2_score,- roc_auc_score,- accuracy_score,- mean_absolute_error,- log_loss,- average_precision_score,- f1_score,- mean_absolute_percentage_error,- ndcg_score,-)-from sklearn.model_selection import RepeatedStratifiedKFold, GroupKFold, TimeSeriesSplit-from .model import (- XGBoostSklearnEstimator,- XGBoost_TS,- XGBoostLimitDepthEstimator,- XGBoostLimitDepth_TS,- RandomForestEstimator,- RF_TS,- LGBMEstimator,- LGBM_TS,- LRL1Classifier,- LRL2Classifier,- CatBoostEstimator,- ExtraTreesEstimator,- ExtraTrees_TS,- KNeighborsEstimator,- Prophet,- ARIMA,- SARIMAX,- TransformersEstimator,- TemporalFusionTransformerEstimator,- TransformersEstimatorModelSelection,-)-from .data import CLASSIFICATION, group_counts, TS_FORECAST-import logging--logger = logging.getLogger(__name__)--sklearn_metric_name_set = {- "r2",- "rmse",- "mae",- "mse",- "accuracy",- "roc_auc",- "roc_auc_ovr",- "roc_auc_ovo",- "roc_auc_weighted",- "roc_auc_ovr_weighted",- "roc_auc_ovo_weighted",- "log_loss",- "mape",- "f1",- "ap",- "ndcg",- "micro_f1",- "macro_f1",-}-huggingface_metric_to_mode = {- "accuracy": "max",- "bertscore": "max",- "bleu": "max",- "bleurt": "max",- "cer": "min",- "chrf": "min",- "code_eval": "max",- "comet": "max",- "competition_math": "max",- "coval": "max",- "cuad": "max",- "f1": "max",- "gleu": "max",- "google_bleu": "max",- "matthews_correlation": "max",- "meteor": "max",- "pearsonr": "max",- "precision": "max",- "recall": "max",- "rouge": "max",- "sacrebleu": "max",- "sari": "max",- "seqeval": "max",- "spearmanr": "max",- "ter": "min",- "wer": "min",-}-huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}---def get_estimator_class(task, estimator_name):- # when adding a new learner, need to add an elif branch- if "xgboost" == estimator_name:- estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator- elif "xgb_limitdepth" == estimator_name:- estimator_class = (- XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator- )- elif "rf" == estimator_name:- estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator- elif "lgbm" == estimator_name:- estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator- elif "lrl1" == estimator_name:- estimator_class = LRL1Classifier- elif "lrl2" == estimator_name:- estimator_class = LRL2Classifier- elif "catboost" == estimator_name:- estimator_class = CatBoostEstimator- elif "extra_tree" == estimator_name:- estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator- elif "kneighbor" == estimator_name:- estimator_class = KNeighborsEstimator- elif "prophet" in estimator_name:- estimator_class = Prophet- elif estimator_name == "arima":- estimator_class = ARIMA- elif estimator_name == "sarimax":- estimator_class = SARIMAX- elif estimator_name == "transformer":- estimator_class = TransformersEstimator- elif estimator_name == "tft":- estimator_class = TemporalFusionTransformerEstimator- elif estimator_name == "transformer_ms":- estimator_class = TransformersEstimatorModelSelection- else:- raise ValueError(- estimator_name + " is not a built-in learner. "- "Please use AutoML.add_learner() to add a customized learner."- )- return estimator_class---def metric_loss_score(- metric_name,- y_processed_predict,- y_processed_true,- labels=None,- sample_weight=None,- groups=None,-):- # y_processed_predict and y_processed_true are processed id labels if the original were the token labels- if is_in_sklearn_metric_name_set(metric_name):- return sklearn_metric_loss_score(- metric_name,- y_processed_predict,- y_processed_true,- labels,- sample_weight,- groups,- )- else:- try:- import datasets-- datasets_metric_name = huggingface_submetric_to_metric.get(- metric_name, metric_name.split(":")[0]- )- metric = datasets.load_metric(datasets_metric_name)- metric_mode = huggingface_metric_to_mode[datasets_metric_name]-- if metric_name.startswith("seqeval"):- y_processed_true = [- [labels[tr] for tr in each_list] for each_list in y_processed_true- ]- elif metric in ("pearsonr", "spearmanr"):- y_processed_true = (- y_processed_true.to_list()- if isinstance(y_processed_true, pd.Series)- else list(y_processed_true)- )- score_dict = metric.compute(- predictions=y_processed_predict, references=y_processed_true- )- if "rouge" in metric_name:- score = score_dict[metric_name].mid.fmeasure- elif metric_name.startswith("seqeval"):- metric_submetric_names = metric_name.split(":")- score = score_dict[- metric_submetric_names[1]- if len(metric_submetric_names) > 1- else "overall_accuracy"- ]- else:- score = score_dict[metric_name]- except ImportError:- raise ValueError(- metric_name- + " is not an built-in sklearn metric and nlp is not installed. "- "Currently built-in sklearn metrics are: "- "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"- "log_loss, mape, f1, micro_f1, macro_f1, ap. "- "If the metric is an nlp metric, please pip install flaml[nlp] ",- "or pass a customized metric function to AutoML.fit(metric=func)",- )- # If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)- # ask the user to provide a custom metric- except FileNotFoundError:- raise ValueError(- metric_name + " is neither an sklearn metric nor a huggingface metric. "- "Currently built-in sklearn metrics are: "- "r2, rmse, mae, mse, accuracy, roc_auc, roc_auc_ovr, roc_auc_ovo,"- "log_loss, mape, f1, micro_f1, macro_f1, ap. "- "Currently built-in huggingface metrics are: "- + ", ".join(huggingface_metric_to_mode.keys())- + ". Please pass a customized metric function to AutoML.fit(metric=func)"- )- if metric_mode == "max":- return 1 - score- else:- return score---def is_in_sklearn_metric_name_set(metric_name):- return metric_name.startswith("ndcg") or metric_name in sklearn_metric_name_set---def is_min_metric(metric_name):- return (- metric_name in ["rmse", "mae", "mse", "log_loss", "mape"]- or huggingface_metric_to_mode.get(metric_name, None) == "min"- )---def sklearn_metric_loss_score(- metric_name,- y_predict,- y_true,- labels=None,- sample_weight=None,- groups=None,-):- """Loss using the specified metric.-- Args:- metric_name: A string of the metric name, one of- 'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'roc_auc_ovr',- 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted',- 'log_loss', 'mape', 'f1', 'ap', 'ndcg', 'micro_f1', 'macro_f1'.- y_predict: A 1d or 2d numpy array of the predictions which can be- used to calculate the metric. E.g., 2d for log_loss and 1d- for others.- y_true: A 1d numpy array of the true labels.- labels: A list or an array of the unique labels.- sample_weight: A 1d numpy array of the sample weight.- groups: A 1d numpy array of the group labels.-- Returns:- score: A float number of the loss, the lower the better.- """- metric_name = metric_name.lower()-- if "r2" == metric_name:- score = 1.0 - r2_score(y_true, y_predict, sample_weight=sample_weight)- elif metric_name == "rmse":- score = np.sqrt(- mean_squared_error(y_true, y_predict, sample_weight=sample_weight)- )- elif metric_name == "mae":- score = mean_absolute_error(y_true, y_predict, sample_weight=sample_weight)- elif metric_name == "mse":- score = mean_squared_error(y_true, y_predict, sample_weight=sample_weight)- elif metric_name == "accuracy":- score = 1.0 - accuracy_score(y_true, y_predict, sample_weight=sample_weight)- elif metric_name == "roc_auc":- score = 1.0 - roc_auc_score(y_true, y_predict, sample_weight=sample_weight)- elif metric_name == "roc_auc_ovr":- score = 1.0 - roc_auc_score(- y_true, y_predict, sample_weight=sample_weight, multi_class="ovr"- )- elif metric_name == "roc_auc_ovo":- score = 1.0 - roc_auc_score(- y_true, y_predict, sample_weight=sample_weight, multi_class="ovo"- )- elif metric_name == "roc_auc_weighted":- score = 1.0 - roc_auc_score(- y_true, y_predict, sample_weight=sample_weight, average="weighted"- )- elif metric_name == "roc_auc_ovo_weighted":- score = 1.0 - roc_auc_score(- y_true,- y_predict,- sample_weight=sample_weight,- average="weighted",- multi_class="ovo",- )- elif metric_name == "roc_auc_ovr_weighted":- score = 1.0 - roc_auc_score(- y_true,- y_predict,- sample_weight=sample_weight,- average="weighted",- multi_class="ovr",- )- elif "log_loss" == metric_name:- score = log_loss(y_true, y_predict, labels=labels, sample_weight=sample_weight)- elif "mape" == metric_name:- try:- score = mean_absolute_percentage_error(y_true, y_predict)- except ValueError:- return np.inf- elif "micro_f1" == metric_name:- score = 1 - f1_score(- y_true, y_predict, sample_weight=sample_weight, average="micro"- )- elif "macro_f1" == metric_name:- score = 1 - f1_score(- y_true, y_predict, sample_weight=sample_weight, average="macro"- )- elif "f1" == metric_name:- score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight)- elif "ap" == metric_name:- score = 1 - average_precision_score(- y_true, y_predict, sample_weight=sample_weight- )- elif "ndcg" in metric_name:- if "@" in metric_name:- k = int(metric_name.split("@", 1)[-1])- counts = group_counts(groups)- score = 0- psum = 0- for c in counts:- score -= ndcg_score(- np.asarray([y_true[psum : psum + c]]),- np.asarray([y_predict[psum : psum + c]]),- k=k,- )- psum += c- score /= len(counts)- score += 1- else:- score = 1 - ndcg_score([y_true], [y_predict])- return score---def get_y_pred(estimator, X, eval_metric, obj):- if eval_metric in ["roc_auc", "ap", "roc_auc_weighted"] and "binary" in obj:- y_pred_classes = estimator.predict_proba(X)- y_pred = y_pred_classes[:, 1] if y_pred_classes.ndim > 1 else y_pred_classes- elif eval_metric in [- "log_loss",- "roc_auc",- "roc_auc_ovr",- "roc_auc_ovo",- "roc_auc_ovo_weighted",- "roc_auc_ovr_weighted",- ]:- y_pred = estimator.predict_proba(X)- else:- y_pred = estimator.predict(X)- return y_pred---def _eval_estimator(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- obj,- labels=None,- log_training_metric=False,- fit_kwargs={},-):- if isinstance(eval_metric, str):- pred_start = time.time()- val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)- pred_time = (time.time() - pred_start) / X_val.shape[0]-- val_loss = metric_loss_score(- eval_metric,- y_processed_predict=val_pred_y,- y_processed_true=y_val,- labels=labels,- sample_weight=weight_val,- groups=groups_val,- )- metric_for_logging = {"pred_time": pred_time}- if log_training_metric:- train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj)- metric_for_logging["train_loss"] = metric_loss_score(- eval_metric,- train_pred_y,- y_train,- labels,- fit_kwargs.get("sample_weight"),- fit_kwargs.get("groups"),- )- else: # customized metric function- val_loss, metric_for_logging = eval_metric(- X_val,- y_val,- estimator,- labels,- X_train,- y_train,- weight_val,- fit_kwargs.get("sample_weight"),- config,- groups_val,- fit_kwargs.get("groups"),- )- pred_time = metric_for_logging.get("pred_time", 0)- val_pred_y = None- # eval_metric may return val_pred_y but not necessarily. Setting None for now.- return val_loss, metric_for_logging, pred_time, val_pred_y---def get_val_loss(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- obj,- labels=None,- budget=None,- log_training_metric=False,- fit_kwargs={},- free_mem_ratio=0,-):-- start = time.time()- # if groups_val is not None:- # fit_kwargs['groups_val'] = groups_val- # fit_kwargs['X_val'] = X_val- # fit_kwargs['y_val'] = y_val- estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)- val_loss, metric_for_logging, pred_time, _ = _eval_estimator(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- obj,- labels,- log_training_metric,- fit_kwargs,- )- if hasattr(estimator, "intermediate_results"):- metric_for_logging["intermediate_results"] = estimator.intermediate_results- train_time = time.time() - start- return val_loss, metric_for_logging, train_time, pred_time---def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):- metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)- metrics_to_log = None- for single_fold in log_metrics_folds:- if metrics_to_log is None:- metrics_to_log = single_fold- elif isinstance(metrics_to_log, dict):- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}- else:- metrics_to_log += single_fold- if metrics_to_log:- n = len(val_loss_folds)- metrics_to_log = (- {k: v / n for k, v in metrics_to_log.items()}- if isinstance(metrics_to_log, dict)- else metrics_to_log / n- )- return metric_to_minimize, metrics_to_log---def evaluate_model_CV(- config,- estimator,- X_train_all,- y_train_all,- budget,- kf,- task,- eval_metric,- best_val_loss,- cv_score_agg_func=None,- log_training_metric=False,- fit_kwargs={},- free_mem_ratio=0,-):- if cv_score_agg_func is None:- cv_score_agg_func = default_cv_score_agg_func- start_time = time.time()- val_loss_folds = []- log_metric_folds = []- metric = None- train_time = pred_time = 0- total_fold_num = 0- n = kf.get_n_splits()- X_train_split, y_train_split = X_train_all, y_train_all- if task in CLASSIFICATION:- labels = np.unique(y_train_all)- else:- labels = fit_kwargs.get(- "label_list"- ) # pass the label list on to compute the evaluation metric- groups = None- shuffle = getattr(kf, "shuffle", task not in TS_FORECAST)- if isinstance(kf, RepeatedStratifiedKFold):- kf = kf.split(X_train_split, y_train_split)- elif isinstance(kf, GroupKFold):- groups = kf.groups- kf = kf.split(X_train_split, y_train_split, groups)- shuffle = False- elif isinstance(kf, TimeSeriesSplit):- kf = kf.split(X_train_split, y_train_split)- else:- kf = kf.split(X_train_split)- rng = np.random.RandomState(2020)- budget_per_train = budget and budget / n- if "sample_weight" in fit_kwargs:- weight = fit_kwargs["sample_weight"]- weight_val = None- else:- weight = weight_val = None- for train_index, val_index in kf:- if shuffle:- train_index = rng.permutation(train_index)- if isinstance(X_train_all, pd.DataFrame):- X_train = X_train_split.iloc[train_index]- X_val = X_train_split.iloc[val_index]- else:- X_train, X_val = X_train_split[train_index], X_train_split[val_index]- y_train, y_val = y_train_split[train_index], y_train_split[val_index]- estimator.cleanup()- if weight is not None:- fit_kwargs["sample_weight"], weight_val = (- weight[train_index],- weight[val_index],- )- if groups is not None:- fit_kwargs["groups"] = groups[train_index]- groups_val = groups[val_index]- else:- groups_val = None- val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- task,- labels,- budget_per_train,- log_training_metric=log_training_metric,- fit_kwargs=fit_kwargs,- free_mem_ratio=free_mem_ratio,- )- if isinstance(metric_i, dict) and "intermediate_results" in metric_i.keys():- del metric_i["intermediate_results"]- if weight is not None:- fit_kwargs["sample_weight"] = weight- total_fold_num += 1- val_loss_folds.append(val_loss_i)- log_metric_folds.append(metric_i)- train_time += train_time_i- pred_time += pred_time_i- if budget and time.time() - start_time >= budget:- break- val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)- n = total_fold_num- pred_time /= n- return val_loss, metric, train_time, pred_time---def compute_estimator(- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- budget,- kf,- config_dic,- task,- estimator_name,- eval_method,- eval_metric,- best_val_loss=np.Inf,- n_jobs=1,- estimator_class=None,- cv_score_agg_func=None,- log_training_metric=False,- fit_kwargs={},- free_mem_ratio=0,-):- estimator_class = estimator_class or get_estimator_class(task, estimator_name)- estimator = estimator_class(- **config_dic,- task=task,- n_jobs=n_jobs,- )-- if isinstance(estimator, TransformersEstimator):- # TODO: move the partial function to nlp- fit_kwargs["metric"] = eval_metric- fit_kwargs["X_val"] = X_val- fit_kwargs["y_val"] = y_val-- if "holdout" == eval_method:- val_loss, metric_for_logging, train_time, pred_time = get_val_loss(- config_dic,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- task,- labels=fit_kwargs.get(- "label_list"- ), # pass the label list on to compute the evaluation metric- budget=budget,- log_training_metric=log_training_metric,- fit_kwargs=fit_kwargs,- free_mem_ratio=0,- )- else:- val_loss, metric_for_logging, train_time, pred_time = evaluate_model_CV(- config_dic,- estimator,- X_train,- y_train,- budget,- kf,- task,- eval_metric,- best_val_loss,- cv_score_agg_func,- log_training_metric=log_training_metric,- fit_kwargs=fit_kwargs,- free_mem_ratio=0,- )-- if isinstance(estimator, TransformersEstimator):- del fit_kwargs["metric"], fit_kwargs["X_val"], fit_kwargs["y_val"]-- return estimator, val_loss, metric_for_logging, train_time, pred_time---def train_estimator(- config_dic,- X_train,- y_train,- task,- estimator_name,- n_jobs=1,- estimator_class=None,- budget=None,- fit_kwargs={},- eval_metric=None,- free_mem_ratio=0,-):- start_time = time.time()- estimator_class = estimator_class or get_estimator_class(task, estimator_name)- estimator = estimator_class(- **config_dic,- task=task,- n_jobs=n_jobs,- )- if isinstance(estimator, TransformersEstimator):- fit_kwargs["metric"] = eval_metric-- if X_train is not None:- train_time = estimator.fit(- X_train, y_train, budget, free_mem_ratio, **fit_kwargs- )- else:- estimator = estimator.estimator_class(**estimator.params)- train_time = time.time() - start_time- return estimator, train_time---def get_classification_objective(num_labels: int) -> str:- if num_labels == 2:- objective_name = "binary"- else:- objective_name = "multiclass"- return objective_name---def norm_confusion_matrix(y_true, y_pred):- """normalized confusion matrix.-- Args:- estimator: A multi-class classification estimator.- y_true: A numpy array or a pandas series of true labels.- y_pred: A numpy array or a pandas series of predicted labels.-- Returns:- A normalized confusion matrix.- """- from sklearn.metrics import confusion_matrix-- conf_mat = confusion_matrix(y_true, y_pred)- norm_conf_mat = conf_mat.astype("float") / conf_mat.sum(axis=1)[:, np.newaxis]- return norm_conf_mat---def multi_class_curves(y_true, y_pred_proba, curve_func):- """Binarize the data for multi-class tasks and produce ROC or precision-recall curves.-- Args:- y_true: A numpy array or a pandas series of true labels.- y_pred_proba: A numpy array or a pandas dataframe of predicted probabilites.- curve_func: A function to produce a curve (e.g., roc_curve or precision_recall_curve).-- Returns:- A tuple of two dictionaries with the same set of keys (class indices).- The first dictionary curve_x stores the x coordinates of each curve, e.g.,- curve_x[0] is an 1D array of the x coordinates of class 0.- The second dictionary curve_y stores the y coordinates of each curve, e.g.,- curve_y[0] is an 1D array of the y coordinates of class 0.- """- from sklearn.preprocessing import label_binarize-- classes = np.unique(y_true)- y_true_binary = label_binarize(y_true, classes=classes)-- curve_x, curve_y = {}, {}- for i in range(len(classes)):- curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])- return curve_x, curve_y+import warnings++from flaml.automl.ml import *+++warnings.warn(+ "Importing from `flaml.ml` is deprecated. Please use `flaml.automl.ml`.",+ DeprecationWarning,+)diff --git a/flaml/model.py b/flaml/model.pyindex 5fb2d723e..b780a67d1 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -1,2393 +1,9 @@-# !-# * Copyright (c) FLAML authors. All rights reserved.-# * Licensed under the MIT License. See LICENSE file in the-# * project root for license information.-from contextlib import contextmanager-from functools import partial-import signal-import os-from typing import Callable, List-import numpy as np-import time-from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier-from sklearn.ensemble import ExtraTreesRegressor, ExtraTreesClassifier-from sklearn.linear_model import LogisticRegression-from sklearn.dummy import DummyClassifier, DummyRegressor-from scipy.sparse import issparse-import logging-import shutil-from pandas import DataFrame, Series, to_datetime-import sys-import math-from . import tune-from .data import (- group_counts,- CLASSIFICATION,- add_time_idx_col,- TS_FORECASTREGRESSION,- TS_TIMESTAMP_COL,- TS_VALUE_COL,- SEQCLASSIFICATION,- SEQREGRESSION,- TOKENCLASSIFICATION,- SUMMARIZATION,- NLG_TASKS,-)--try:- import psutil-except ImportError:- psutil = None-try:- import resource-except ImportError:- resource = None--logger = logging.getLogger("flaml.automl")-# FREE_MEM_RATIO = 0.2---def TimeoutHandler(sig, frame):- raise TimeoutError(sig, frame)---@contextmanager-def limit_resource(memory_limit, time_limit):- if memory_limit > 0:- soft, hard = resource.getrlimit(resource.RLIMIT_AS)- if soft < 0 and (hard < 0 or memory_limit <= hard) or memory_limit < soft:- try:- resource.setrlimit(resource.RLIMIT_AS, (int(memory_limit), hard))- except ValueError:- # According to https://bugs.python.org/issue40518, it's a mac-specific error.- pass- main_thread = False- if time_limit is not None:- try:- signal.signal(signal.SIGALRM, TimeoutHandler)- signal.alarm(int(time_limit) or 1)- main_thread = True- except ValueError:- pass- try:- yield- finally:- if main_thread:- signal.alarm(0)- if memory_limit > 0:- resource.setrlimit(resource.RLIMIT_AS, (soft, hard))---class BaseEstimator:- """The abstract class for all learners.-- Typical examples:- * XGBoostEstimator: for regression.- * XGBoostSklearnEstimator: for classification.- * LGBMEstimator, RandomForestEstimator, LRL1Classifier, LRL2Classifier:- for both regression and classification.- """-- def __init__(self, task="binary", **config):- """Constructor.-- Args:- task: A string of the task type, one of- 'binary', 'multiclass', 'regression', 'rank', 'seq-classification',- 'seq-regression', 'token-classification', 'multichoice-classification',- 'summarization', 'ts_forecast', 'ts_forecast_classification'.- config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.- n_jobs is the number of parallel threads.- """- self._task = task- self.params = self.config2params(config)- self.estimator_class = self._model = None- if "_estimator_type" in config:- self._estimator_type = self.params.pop("_estimator_type")- else:- self._estimator_type = (- "classifier" if task in CLASSIFICATION else "regressor"- )-- def get_params(self, deep=False):- params = self.params.copy()- params["task"] = self._task- if hasattr(self, "_estimator_type"):- params["_estimator_type"] = self._estimator_type- return params-- @property- def classes_(self):- return self._model.classes_-- @property- def n_features_in_(self):- return self._model.n_features_in_-- @property- def model(self):- """Trained model after fit() is called, or None before fit() is called."""- return self._model-- @property- def estimator(self):- """Trained model after fit() is called, or None before fit() is called."""- return self._model-- @property- def feature_names_in_(self):- """- if self._model has attribute feature_names_in_, return it.- otherwise, if self._model has attribute feature_name_, return it.- otherwise, if self._model has attribute feature_names, return it.- otherwise, if self._model has method get_booster, return the feature names.- otherwise, return None.- """- if hasattr(self._model, "feature_names_in_"): # for sklearn, xgboost>=1.6- return self._model.feature_names_in_- if hasattr(self._model, "feature_name_"): # for lightgbm- return self._model.feature_name_- if hasattr(self._model, "feature_names"): # for XGBoostEstimator- return self._model.feature_names- if hasattr(self._model, "get_booster"):- # get feature names for xgboost<1.6- # https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.Booster.feature_names- booster = self._model.get_booster()- return booster.feature_names- return None-- @property- def feature_importances_(self):- """- if self._model has attribute feature_importances_, return it.- otherwise, if self._model has attribute coef_, return it.- otherwise, return None.- """- if hasattr(self._model, "feature_importances_"):- # for sklearn, lightgbm, catboost, xgboost- return self._model.feature_importances_- elif hasattr(self._model, "coef_"): # for linear models- return self._model.coef_- else:- return None-- def _preprocess(self, X):- return X-- def _fit(self, X_train, y_train, **kwargs):-- current_time = time.time()- if "groups" in kwargs:- kwargs = kwargs.copy()- groups = kwargs.pop("groups")- if self._task == "rank":- kwargs["group"] = group_counts(groups)- # groups_val = kwargs.get('groups_val')- # if groups_val is not None:- # kwargs['eval_group'] = [group_counts(groups_val)]- # kwargs['eval_set'] = [- # (kwargs['X_val'], kwargs['y_val'])]- # kwargs['verbose'] = False- # del kwargs['groups_val'], kwargs['X_val'], kwargs['y_val']- X_train = self._preprocess(X_train)- model = self.estimator_class(**self.params)- if logger.level == logging.DEBUG:- # xgboost 1.6 doesn't display all the params in the model str- logger.debug(f"flaml.model - {model} fit started with params {self.params}")- model.fit(X_train, y_train, **kwargs)- if logger.level == logging.DEBUG:- logger.debug(f"flaml.model - {model} fit finished")- train_time = time.time() - current_time- self._model = model- return train_time-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- """Train the model from given training data.-- Args:- X_train: A numpy array or a dataframe of training data in shape n*m.- y_train: A numpy array or a series of labels in shape n*1.- budget: A float of the time budget in seconds.- free_mem_ratio: A float between 0 and 1 for the free memory ratio to keep during training.-- Returns:- train_time: A float of the training time in seconds.- """- if (- getattr(self, "limit_resource", None)- and resource is not None- and (budget is not None or psutil is not None)- ):- start_time = time.time()- mem = psutil.virtual_memory() if psutil is not None else None- try:- with limit_resource(- mem.available * (1 - free_mem_ratio)- + psutil.Process(os.getpid()).memory_info().rss- if mem is not None- else -1,- budget,- ):- train_time = self._fit(X_train, y_train, **kwargs)- except (MemoryError, TimeoutError) as e:- logger.warning(f"{e.__class__} {e}")- if self._task in CLASSIFICATION:- model = DummyClassifier()- else:- model = DummyRegressor()- X_train = self._preprocess(X_train)- model.fit(X_train, y_train)- self._model = model- train_time = time.time() - start_time- else:- train_time = self._fit(X_train, y_train, **kwargs)- return train_time-- def predict(self, X, **kwargs):- """Predict label from features.-- Args:- X: A numpy array or a dataframe of featurized instances, shape n*m.-- Returns:- A numpy array of shape n*1.- Each element is the label for a instance.- """- if self._model is not None:- X = self._preprocess(X)- return self._model.predict(X, **kwargs)- else:- logger.warning(- "Estimator is not fit yet. Please run fit() before predict()."- )- return np.ones(X.shape[0])-- def predict_proba(self, X, **kwargs):- """Predict the probability of each class from features.-- Only works for classification problems-- Args:- X: A numpy array of featurized instances, shape n*m.-- Returns:- A numpy array of shape n*c. c is the # classes.- Each element at (i,j) is the probability for instance i to be in- class j.- """- assert self._task in CLASSIFICATION, "predict_proba() only for classification."-- X = self._preprocess(X)- return self._model.predict_proba(X, **kwargs)-- def score(self, X_val: DataFrame, y_val: Series, **kwargs):- """Report the evaluation score of a trained estimator.--- Args:- X_val: A pandas dataframe of the validation input data.- y_val: A pandas series of the validation label.- kwargs: keyword argument of the evaluation function, for example:- - metric: A string of the metric name or a function- e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',- 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',- 'mape'. Default is 'auto'.- If metric is given, the score will report the user specified metric.- If metric is not given, the metric is set to accuracy for classification and r2- for regression.- You can also pass a customized metric function, for examples on how to pass a- customized metric function, please check- [test/nlp/test_autohf_custom_metric.py](https://github.com/microsoft/FLAML/blob/main/test/nlp/test_autohf_custom_metric.py) and- [test/automl/test_multiclass.py](https://github.com/microsoft/FLAML/blob/main/test/automl/test_multiclass.py).-- Returns:- The evaluation score on the validation dataset.- """- from .ml import metric_loss_score- from .ml import is_min_metric-- if self._model is not None:- if self._task == "rank":- raise NotImplementedError(- "AutoML.score() is not implemented for ranking"- )- else:- X_val = self._preprocess(X_val)- metric = kwargs.pop("metric", None)- if metric:- y_pred = self.predict(X_val, **kwargs)- if is_min_metric(metric):- return metric_loss_score(metric, y_pred, y_val)- else:- return 1.0 - metric_loss_score(metric, y_pred, y_val)- else:- return self._model.score(X_val, y_val, **kwargs)- else:- logger.warning(- "Estimator is not fit yet. Please run fit() before predict()."- )- return 0.0-- def cleanup(self):- del self._model- self._model = None-- @classmethod- def search_space(cls, data_size, task, **params):- """[required method] search space.-- Args:- data_size: A tuple of two integers, number of rows and columns.- task: A str of the task type, e.g., "binary", "multiclass", "regression".-- Returns:- A dictionary of the search space.- Each key is the name of a hyperparameter, and value is a dict with- its domain (required) and low_cost_init_value, init_value,- cat_hp_cost (if applicable).- e.g., ```{'domain': tune.randint(lower=1, upper=10), 'init_value': 1}```.- """- return {}-- @classmethod- def size(cls, config: dict) -> float:- """[optional method] memory size of the estimator in bytes.-- Args:- config: A dict of the hyperparameter config.-- Returns:- A float of the memory size required by the estimator to train the- given config.- """- return 1.0-- @classmethod- def cost_relative2lgbm(cls) -> float:- """[optional method] relative cost compared to lightgbm."""- return 1.0-- @classmethod- def init(cls):- """[optional method] initialize the class."""- pass-- def config2params(self, config: dict) -> dict:- """[optional method] config dict to params dict-- Args:- config: A dict of the hyperparameter config.-- Returns:- A dict that will be passed to self.estimator_class's constructor.- """- params = config.copy()- if "FLAML_sample_size" in params:- params.pop("FLAML_sample_size")- return params---class TransformersEstimator(BaseEstimator):- """The class for fine-tuning language models, using huggingface transformers API."""-- ITER_HP = "global_max_steps"-- def __init__(self, task="seq-classification", **config):- super().__init__(task, **config)- import uuid-- self.trial_id = str(uuid.uuid1().hex)[:8]- if task not in NLG_TASKS: # TODO: not in NLG_TASKS- from .nlp.huggingface.training_args import (- TrainingArgumentsForAuto as TrainingArguments,- )- else:- from .nlp.huggingface.training_args import (- Seq2SeqTrainingArgumentsForAuto as TrainingArguments,- )- self._TrainingArguments = TrainingArguments-- @classmethod- def search_space(cls, data_size, task, **params):- search_space_dict = {- "learning_rate": {- "domain": tune.loguniform(1e-6, 1e-4),- "init_value": 1e-5,- },- "num_train_epochs": {- "domain": tune.choice([1, 2, 3, 4, 5]),- "init_value": 3, # to be consistent with roberta- "low_cost_init_value": 1,- },- "per_device_train_batch_size": {- "domain": tune.choice([4, 8, 16, 32, 64]),- "init_value": 32,- "low_cost_init_value": 64,- },- "seed": {- "domain": tune.choice(range(1, 40)),- "init_value": 20,- },- "global_max_steps": {- "domain": sys.maxsize,- "init_value": sys.maxsize,- },- }-- return search_space_dict-- @property- def fp16(self):- return self._kwargs.get("gpu_per_trial") and self._training_args.fp16-- @property- def no_cuda(self):- return not self._kwargs.get("gpu_per_trial")-- def _set_training_args(self, **kwargs):- from .nlp.utils import date_str, Counter-- for (key, val) in kwargs.items():- assert key not in self.params, (- "Since {} is in the search space, it cannot exist in 'custom_fit_kwargs' at the same time."- "If you need to fix the value of {} to {}, the only way is to add a single-value domain in the search "- "space by adding:\n '{}': {{ 'domain': {} }} to 'custom_hp'. For example:"- 'automl_settings["custom_hp"] = {{ "transformer": {{ "model_path": {{ "domain" : '- '"google/electra-small-discriminator" }} }} }}'.format(- key, key, val, key, val- )- )-- """- If use has specified any custom args for TrainingArguments, update these arguments- """- self._training_args = self._TrainingArguments(**kwargs)-- """- Update the attributes in TrainingArguments with self.params values- """- for key, val in self.params.items():- if hasattr(self._training_args, key):- setattr(self._training_args, key, val)-- """- Update the attributes in TrainingArguments that depends on the values of self.params- """- local_dir = os.path.join(- self._training_args.output_dir, "train_{}".format(date_str())- )- if self._use_ray is True:- import ray-- self._training_args.output_dir = ray.tune.get_trial_dir()- else:- self._training_args.output_dir = Counter.get_trial_fold_name(- local_dir, self.params, self.trial_id- )-- self._training_args.fp16 = self.fp16- self._training_args.no_cuda = self.no_cuda-- if (- self._task == TOKENCLASSIFICATION- and self._training_args.max_seq_length is not None- ):- logger.warning(- "For token classification task, FLAML currently does not support customizing the max_seq_length, max_seq_length will be reset to None."- )- setattr(self._training_args, "max_seq_length", None)-- def _tokenize_text(self, X, y=None, **kwargs):- from .nlp.huggingface.utils import tokenize_text- from .nlp.utils import is_a_list_of_str-- is_str = str(X.dtypes[0]) in ("string", "str")- is_list_of_str = is_a_list_of_str(X[list(X.keys())[0]].to_list()[0])-- if is_str or is_list_of_str:- return tokenize_text(- X=X,- Y=y,- task=self._task,- hf_args=self._training_args,- tokenizer=self.tokenizer,- )- else:- return X, y-- def _model_init(self):- from .nlp.huggingface.utils import load_model-- this_model = load_model(- checkpoint_path=self._training_args.model_path,- task=self._task,- num_labels=self.num_labels,- )- return this_model-- def _preprocess_data(self, X, y):- from datasets import Dataset-- processed_X, processed_y_df = self._tokenize_text(X=X, y=y, **self._kwargs)- # convert y from pd.DataFrame back to pd.Series- processed_y = processed_y_df.iloc[:, 0]-- processed_dataset = Dataset.from_pandas(processed_X.join(processed_y_df))-- return processed_dataset, processed_X, processed_y-- @property- def num_labels(self):- if self._task == SEQREGRESSION:- return 1- elif self._task == SEQCLASSIFICATION:- return len(set(self._y_train))- elif self._task == TOKENCLASSIFICATION:- return len(self._training_args.label_list)- else:- return None-- @property- def tokenizer(self):- from transformers import AutoTokenizer-- if self._task == SUMMARIZATION:- return AutoTokenizer.from_pretrained(- pretrained_model_name_or_path=self._training_args.model_path,- cache_dir=None,- use_fast=True,- revision="main",- use_auth_token=None,- )- else:- return AutoTokenizer.from_pretrained(- self._training_args.model_path,- use_fast=True,- add_prefix_space=self._add_prefix_space,- )-- @property- def data_collator(self):- from .nlp.huggingface.data_collator import task_to_datacollator_class-- data_collator_class = task_to_datacollator_class.get(self._task)-- if data_collator_class:- kwargs = {- "model": self._model_init(),- # need to set model, or there's ValueError: Expected input batch_size (..) to match target batch_size (..)- "label_pad_token_id": -100, # pad with token id -100- "pad_to_multiple_of": 8,- # pad to multiple of 8 because quote Transformers: "This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta)"- "tokenizer": self.tokenizer,- }-- for key in list(kwargs.keys()):- if (- key not in data_collator_class.__dict__.keys()- and key != "tokenizer"- ):- del kwargs[key]- return data_collator_class(**kwargs)- else:- return None-- def fit(- self,- X_train: DataFrame,- y_train: Series,- budget=None,- free_mem_ratio=0,- X_val=None,- y_val=None,- gpu_per_trial=None,- metric=None,- **kwargs,- ):- import transformers-- transformers.logging.set_verbosity_error()-- from transformers import TrainerCallback- from transformers.trainer_utils import set_seed- from .nlp.huggingface.trainer import TrainerForAuto-- try:- from ray.tune import is_session_enabled-- self._use_ray = is_session_enabled()- except ImportError:- self._use_ray = False-- this_params = self.params- self._kwargs = kwargs-- self._X_train, self._y_train = X_train, y_train- self._set_training_args(**kwargs)- self._add_prefix_space = (- "roberta" in self._training_args.model_path- ) # If using roberta model, must set add_prefix_space to True to avoid the assertion error at- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249-- train_dataset, self._X_train, self._y_train = self._preprocess_data(- X_train, y_train- )- if X_val is not None:- eval_dataset, self._X_val, self._y_val = self._preprocess_data(X_val, y_val)- else:- eval_dataset, self._X_val, self._y_val = None, None, None-- set_seed(self.params.get("seed", self._training_args.seed))- self._metric = metric-- class EarlyStoppingCallbackForAuto(TrainerCallback):- def on_train_begin(self, args, state, control, **callback_kwargs):- self.train_begin_time = time.time()-- def on_step_begin(self, args, state, control, **callback_kwargs):- self.step_begin_time = time.time()-- def on_step_end(self, args, state, control, **callback_kwargs):- if state.global_step == 1:- self.time_per_iter = time.time() - self.step_begin_time- if (- budget- and (- time.time() + self.time_per_iter- > self.train_begin_time + budget- )- or state.global_step >= this_params[TransformersEstimator.ITER_HP]- ):- control.should_training_stop = True- control.should_save = True- control.should_evaluate = True- return control-- def on_epoch_end(self, args, state, control, **callback_kwargs):- if (- control.should_training_stop- or state.epoch + 1 >= args.num_train_epochs- ):- control.should_save = True- control.should_evaluate = True-- self._trainer = TrainerForAuto(- args=self._training_args,- model_init=self._model_init,- train_dataset=train_dataset,- eval_dataset=eval_dataset,- tokenizer=self.tokenizer,- data_collator=self.data_collator,- compute_metrics=self._compute_metrics_by_dataset_name,- callbacks=[EarlyStoppingCallbackForAuto],- )-- if self._task in NLG_TASKS:- setattr(self._trainer, "_is_seq2seq", True)-- """- When not using ray for tuning, set the limit of CUDA_VISIBLE_DEVICES to math.ceil(gpu_per_trial),- so each estimator does not see all the GPUs- """- if gpu_per_trial is not None:- tmp_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "")- self._trainer.args._n_gpu = gpu_per_trial-- # if gpu_per_trial == 0:- # os.environ["CUDA_VISIBLE_DEVICES"] = ""- if tmp_cuda_visible_devices.count(",") != math.ceil(gpu_per_trial) - 1:- os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(- [str(x) for x in range(math.ceil(gpu_per_trial))]- )-- import time-- start_time = time.time()- self._trainer.train()-- if gpu_per_trial is not None:- os.environ["CUDA_VISIBLE_DEVICES"] = tmp_cuda_visible_devices-- self.params[self.ITER_HP] = self._trainer.state.global_step-- self._checkpoint_path = self._select_checkpoint(self._trainer)- self._ckpt_remains = list(self._trainer.ckpt_to_metric.keys())-- if hasattr(self._trainer, "intermediate_results"):- self.intermediate_results = [- x[1]- for x in sorted(- self._trainer.intermediate_results.items(), key=lambda x: x[0]- )- ]- self._trainer = None-- return time.time() - start_time-- def _delete_one_ckpt(self, ckpt_location):- if self._use_ray is False:- try:- shutil.rmtree(ckpt_location)- except FileNotFoundError:- logger.warning("checkpoint {} not found".format(ckpt_location))-- def cleanup(self):- super().cleanup()- if hasattr(self, "_ckpt_remains"):- for each_ckpt in self._ckpt_remains:- self._delete_one_ckpt(each_ckpt)-- def _select_checkpoint(self, trainer):- from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR-- if trainer.ckpt_to_metric:- best_ckpt, _ = min(- trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_automl_metric"]- )- best_ckpt_global_step = trainer.ckpt_to_global_step[best_ckpt]- for each_ckpt in list(trainer.ckpt_to_metric):- if each_ckpt != best_ckpt:- del trainer.ckpt_to_metric[each_ckpt]- del trainer.ckpt_to_global_step[each_ckpt]- self._delete_one_ckpt(each_ckpt)- else:- best_ckpt_global_step = trainer.state.global_step- best_ckpt = os.path.join(- trainer.args.output_dir,- f"{PREFIX_CHECKPOINT_DIR}-{best_ckpt_global_step}",- )- self.params[self.ITER_HP] = best_ckpt_global_step- logger.debug(trainer.state.global_step)- logger.debug(trainer.ckpt_to_global_step)- return best_ckpt-- def _compute_metrics_by_dataset_name(self, eval_pred):- # TODO: call self._metric(eval_pred, self)- if isinstance(self._metric, str):- from .ml import metric_loss_score- from .nlp.huggingface.utils import postprocess_prediction_and_true-- predictions, y_true = eval_pred- # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text- processed_predictions, processed_y_true = postprocess_prediction_and_true(- task=self._task,- y_pred=predictions,- tokenizer=self.tokenizer,- hf_args=self._training_args,- y_true=y_true,- )- metric_dict = {- "automl_metric": metric_loss_score(- metric_name=self._metric,- y_processed_predict=processed_predictions,- y_processed_true=processed_y_true,- labels=self._training_args.label_list,- )- }- else:- # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)- loss, metric_dict = self._metric(- X_test=self._X_val,- y_test=self._y_val,- estimator=self,- labels=None,- X_train=self._X_train,- y_train=self._y_train,- )- metric_dict["automl_metric"] = loss-- return metric_dict-- def _init_model_for_predict(self):- from .nlp.huggingface.trainer import TrainerForAuto-- """- Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistent- with HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947- """- training_args = self._TrainingArguments(- local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16- )- for key, val in self._training_args.__dict__.items():- if key not in ("local_rank", "model_path", "fp16"):- setattr(training_args, key, val)- self._training_args = training_args-- new_trainer = TrainerForAuto(- model=self._model_init(),- args=self._training_args,- data_collator=self.data_collator,- compute_metrics=self._compute_metrics_by_dataset_name,- )- if self._task in NLG_TASKS:- setattr(new_trainer, "_is_seq2seq", True)- return new_trainer-- def predict_proba(self, X, **pred_kwargs):- from datasets import Dataset-- if pred_kwargs:- for key, val in pred_kwargs.items():- setattr(self._training_args, key, val)-- assert (- self._task in CLASSIFICATION- ), "predict_proba() only for classification tasks."-- X_test, _ = self._tokenize_text(X, **self._kwargs)- test_dataset = Dataset.from_pandas(X_test)-- new_trainer = self._init_model_for_predict()- predictions = new_trainer.predict(test_dataset)- return predictions.predictions-- def score(self, X_val: DataFrame, y_val: Series, **kwargs):- import transformers-- transformers.logging.set_verbosity_error()-- self._metric = kwargs["metric"]-- eval_dataset, X_val, y_val = self._preprocess_data(X_val, y_val)-- new_trainer = self._init_model_for_predict()- return new_trainer.evaluate(eval_dataset)-- def predict(self, X, **pred_kwargs):- import transformers- from datasets import Dataset- from .nlp.huggingface.utils import postprocess_prediction_and_true-- transformers.logging.set_verbosity_error()-- if pred_kwargs:- for key, val in pred_kwargs.items():- setattr(self._training_args, key, val)-- X_test, _ = self._tokenize_text(X, **self._kwargs)- test_dataset = Dataset.from_pandas(X_test)-- new_trainer = self._init_model_for_predict()-- if self._task not in NLG_TASKS:- predictions = new_trainer.predict(test_dataset)- else:- predictions = new_trainer.predict(- test_dataset,- metric_key_prefix="predict",- )- post_y_pred, _ = postprocess_prediction_and_true(- task=self._task,- y_pred=predictions.predictions,- tokenizer=self.tokenizer,- hf_args=self._training_args,- X=X,- )- return post_y_pred-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- params[TransformersEstimator.ITER_HP] = params.get(- TransformersEstimator.ITER_HP, sys.maxsize- )- return params---class TransformersEstimatorModelSelection(TransformersEstimator):- def __init__(self, task="seq-classification", **config):- super().__init__(task, **config)-- @classmethod- def search_space(cls, data_size, task, **params):- search_space_dict = TransformersEstimator.search_space(- data_size, task, **params- )-- """- For model selection, use the same search space regardless of memory constraint- If OOM, user should change the search space themselves- """-- search_space_dict["model_path"] = {- "domain": tune.choice(- [- "google/electra-base-discriminator",- "bert-base-uncased",- "roberta-base",- "facebook/muppet-roberta-base",- "google/electra-small-discriminator",- ]- ),- "init_value": "facebook/muppet-roberta-base",- }- return search_space_dict---class SKLearnEstimator(BaseEstimator):- """- The base class for tuning scikit-learn estimators.-- Subclasses can modify the function signature of ``__init__`` to- ignore the values in ``config`` that are not relevant to the constructor- of their underlying estimator. For example, some regressors in ``scikit-learn``- don't accept the ``n_jobs`` parameter contained in ``config``. For these,- one can add ``n_jobs=None,`` before ``**config`` to make sure ``config`` doesn't- contain an ``n_jobs`` key.- """-- def __init__(self, task="binary", **config):- super().__init__(task, **config)-- def _preprocess(self, X):- if isinstance(X, DataFrame):- cat_columns = X.select_dtypes(include=["category"]).columns- if not cat_columns.empty:- X = X.copy()- X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":- # numpy array is not of numeric dtype- X = DataFrame(X)- for col in X.columns:- if isinstance(X[col][0], str):- X[col] = X[col].astype("category").cat.codes- X = X.to_numpy()- return X---class LGBMEstimator(BaseEstimator):- """The class for tuning LGBM, using sklearn API."""-- ITER_HP = "n_estimators"- HAS_CALLBACK = True- DEFAULT_ITER = 100-- @classmethod- def search_space(cls, data_size, **params):- upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower- return {- "n_estimators": {- "domain": tune.lograndint(lower=4, upper=upper),- "init_value": 4,- "low_cost_init_value": 4,- },- "num_leaves": {- "domain": tune.lograndint(lower=4, upper=upper),- "init_value": 4,- "low_cost_init_value": 4,- },- "min_child_samples": {- "domain": tune.lograndint(lower=2, upper=2**7 + 1),- "init_value": 20,- },- "learning_rate": {- "domain": tune.loguniform(lower=1 / 1024, upper=1.0),- "init_value": 0.1,- },- "log_max_bin": { # log transformed with base 2- "domain": tune.lograndint(lower=3, upper=11),- "init_value": 8,- },- "colsample_bytree": {- "domain": tune.uniform(lower=0.01, upper=1.0),- "init_value": 1.0,- },- "reg_alpha": {- "domain": tune.loguniform(lower=1 / 1024, upper=1024),- "init_value": 1 / 1024,- },- "reg_lambda": {- "domain": tune.loguniform(lower=1 / 1024, upper=1024),- "init_value": 1.0,- },- }-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- if "log_max_bin" in params:- params["max_bin"] = (1 << params.pop("log_max_bin")) - 1- return params-- @classmethod- def size(cls, config):- num_leaves = int(- round(- config.get("num_leaves")- or config.get("max_leaves")- or 1 << config.get("max_depth", 16)- )- )- n_estimators = int(round(config["n_estimators"]))- return (num_leaves * 3 + (num_leaves - 1) * 4 + 1.0) * n_estimators * 8-- def __init__(self, task="binary", **config):- super().__init__(task, **config)- if "verbose" not in self.params:- self.params["verbose"] = -1- if "regression" == task:- from lightgbm import LGBMRegressor-- self.estimator_class = LGBMRegressor- elif "rank" == task:- from lightgbm import LGBMRanker-- self.estimator_class = LGBMRanker- else:- from lightgbm import LGBMClassifier-- self.estimator_class = LGBMClassifier- self._time_per_iter = None- self._train_size = 0- self._mem_per_iter = -1- self.HAS_CALLBACK = self.HAS_CALLBACK and self._callbacks(0, 0, 0) is not None-- def _preprocess(self, X):- if (- not isinstance(X, DataFrame)- and issparse(X)- and np.issubdtype(X.dtype, np.integer)- ):- X = X.astype(float)- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":- # numpy array is not of numeric dtype- X = DataFrame(X)- for col in X.columns:- if isinstance(X[col][0], str):- X[col] = X[col].astype("category").cat.codes- X = X.to_numpy()- return X-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- start_time = time.time()- deadline = start_time + budget if budget else np.inf- n_iter = self.params.get(self.ITER_HP, self.DEFAULT_ITER)- trained = False- if not self.HAS_CALLBACK:- mem0 = psutil.virtual_memory().available if psutil is not None else 1- if (- (- not self._time_per_iter- or abs(self._train_size - X_train.shape[0]) > 4- )- and budget is not None- or self._mem_per_iter < 0- and psutil is not None- ) and n_iter > 1:- self.params[self.ITER_HP] = 1- self._t1 = self._fit(X_train, y_train, **kwargs)- if budget is not None and self._t1 >= budget or n_iter == 1:- return self._t1- mem1 = psutil.virtual_memory().available if psutil is not None else 1- self._mem1 = mem0 - mem1- self.params[self.ITER_HP] = min(n_iter, 4)- self._t2 = self._fit(X_train, y_train, **kwargs)- mem2 = psutil.virtual_memory().available if psutil is not None else 1- self._mem2 = max(mem0 - mem2, self._mem1)- # if self._mem1 <= 0:- # self._mem_per_iter = self._mem2 / (self.params[self.ITER_HP] + 1)- # elif self._mem2 <= 0:- # self._mem_per_iter = self._mem1- # else:- self._mem_per_iter = min(- self._mem1, self._mem2 / self.params[self.ITER_HP]- )- # if self._mem_per_iter <= 1 and psutil is not None:- # n_iter = self.params[self.ITER_HP]- self._time_per_iter = (- (self._t2 - self._t1) / (self.params[self.ITER_HP] - 1)- if self._t2 > self._t1- else self._t1- if self._t1- else 0.001- )- self._train_size = X_train.shape[0]- if (- budget is not None- and self._t1 + self._t2 >= budget- or n_iter == self.params[self.ITER_HP]- ):- # self.params[self.ITER_HP] = n_iter- return time.time() - start_time- trained = True- # logger.debug(mem0)- # logger.debug(self._mem_per_iter)- if n_iter > 1:- max_iter = min(- n_iter,- int(- (budget - time.time() + start_time - self._t1)- / self._time_per_iter- + 1- )- if budget is not None- else n_iter,- int((1 - free_mem_ratio) * mem0 / self._mem_per_iter)- if psutil is not None and self._mem_per_iter > 0- else n_iter,- )- if trained and max_iter <= self.params[self.ITER_HP]:- return time.time() - start_time- # when not trained, train at least one iter- self.params[self.ITER_HP] = max(max_iter, 1)- if self.HAS_CALLBACK:- kwargs_callbacks = kwargs.get("callbacks")- if kwargs_callbacks:- callbacks = kwargs_callbacks + self._callbacks(- start_time, deadline, free_mem_ratio- )- kwargs.pop("callbacks")- else:- callbacks = self._callbacks(start_time, deadline, free_mem_ratio)- if isinstance(self, XGBoostSklearnEstimator):- from xgboost import __version__-- if __version__ >= "1.6.0":- # since xgboost>=1.6.0, callbacks can't be passed in fit()- self.params["callbacks"] = callbacks- callbacks = None- self._fit(- X_train,- y_train,- callbacks=callbacks,- **kwargs,- )- if callbacks is None:- # for xgboost>=1.6.0, pop callbacks to enable pickle- callbacks = self.params.pop("callbacks")- self._model.set_params(callbacks=callbacks[:-1])- best_iteration = (- self._model.get_booster().best_iteration- if isinstance(self, XGBoostSklearnEstimator)- else self._model.best_iteration_- )- if best_iteration is not None:- self._model.set_params(n_estimators=best_iteration + 1)- else:- self._fit(X_train, y_train, **kwargs)- train_time = time.time() - start_time- return train_time-- def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:- return [partial(self._callback, start_time, deadline, free_mem_ratio)]-- def _callback(self, start_time, deadline, free_mem_ratio, env) -> None:- from lightgbm.callback import EarlyStopException-- now = time.time()- if env.iteration == 0:- self._time_per_iter = now - start_time- if now + self._time_per_iter > deadline:- raise EarlyStopException(env.iteration, env.evaluation_result_list)- if psutil is not None:- mem = psutil.virtual_memory()- if mem.available / mem.total < free_mem_ratio:- raise EarlyStopException(env.iteration, env.evaluation_result_list)---class XGBoostEstimator(SKLearnEstimator):- """The class for tuning XGBoost regressor, not using sklearn API."""-- DEFAULT_ITER = 10-- @classmethod- def search_space(cls, data_size, **params):- upper = max(5, min(32768, int(data_size[0]))) # upper must be larger than lower- return {- "n_estimators": {- "domain": tune.lograndint(lower=4, upper=upper),- "init_value": 4,- "low_cost_init_value": 4,- },- "max_leaves": {- "domain": tune.lograndint(lower=4, upper=upper),- "init_value": 4,- "low_cost_init_value": 4,- },- "max_depth": {- "domain": tune.choice([0, 6, 12]),- "init_value": 0,- },- "min_child_weight": {- "domain": tune.loguniform(lower=0.001, upper=128),- "init_value": 1.0,- },- "learning_rate": {- "domain": tune.loguniform(lower=1 / 1024, upper=1.0),- "init_value": 0.1,- },- "subsample": {- "domain": tune.uniform(lower=0.1, upper=1.0),- "init_value": 1.0,- },- "colsample_bylevel": {- "domain": tune.uniform(lower=0.01, upper=1.0),- "init_value": 1.0,- },- "colsample_bytree": {- "domain": tune.uniform(lower=0.01, upper=1.0),- "init_value": 1.0,- },- "reg_alpha": {- "domain": tune.loguniform(lower=1 / 1024, upper=1024),- "init_value": 1 / 1024,- },- "reg_lambda": {- "domain": tune.loguniform(lower=1 / 1024, upper=1024),- "init_value": 1.0,- },- }-- @classmethod- def size(cls, config):- return LGBMEstimator.size(config)-- @classmethod- def cost_relative2lgbm(cls):- return 1.6-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- max_depth = params["max_depth"] = params.get("max_depth", 0)- if max_depth == 0:- params["grow_policy"] = params.get("grow_policy", "lossguide")- params["tree_method"] = params.get("tree_method", "hist")- # params["booster"] = params.get("booster", "gbtree")- params["use_label_encoder"] = params.get("use_label_encoder", False)- if "n_jobs" in config:- params["nthread"] = params.pop("n_jobs")- return params-- def __init__(- self,- task="regression",- **config,- ):- super().__init__(task, **config)- self.params["verbosity"] = 0-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import xgboost as xgb-- start_time = time.time()- deadline = start_time + budget if budget else np.inf- if issparse(X_train):- if xgb.__version__ < "1.6.0":- # "auto" fails for sparse input since xgboost 1.6.0- self.params["tree_method"] = "auto"- else:- X_train = self._preprocess(X_train)- if "sample_weight" in kwargs:- dtrain = xgb.DMatrix(X_train, label=y_train, weight=kwargs["sample_weight"])- else:- dtrain = xgb.DMatrix(X_train, label=y_train)-- objective = self.params.get("objective")- if isinstance(objective, str):- obj = None- else:- obj = objective- if "objective" in self.params:- del self.params["objective"]- _n_estimators = self.params.pop("n_estimators")- callbacks = XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)- if callbacks:- self._model = xgb.train(- self.params,- dtrain,- _n_estimators,- obj=obj,- callbacks=callbacks,- )- self.params["n_estimators"] = self._model.best_iteration + 1- else:- self._model = xgb.train(self.params, dtrain, _n_estimators, obj=obj)- self.params["n_estimators"] = _n_estimators- self.params["objective"] = objective- del dtrain- train_time = time.time() - start_time- return train_time-- def predict(self, X, **kwargs):- import xgboost as xgb-- if not issparse(X):- X = self._preprocess(X)- dtest = xgb.DMatrix(X)- return super().predict(dtest, **kwargs)-- @classmethod- def _callbacks(cls, start_time, deadline, free_mem_ratio):- try:- from xgboost.callback import TrainingCallback- except ImportError: # for xgboost<1.3- return None-- class ResourceLimit(TrainingCallback):- def after_iteration(self, model, epoch, evals_log) -> bool:- now = time.time()- if epoch == 0:- self._time_per_iter = now - start_time- if now + self._time_per_iter > deadline:- return True- if psutil is not None:- mem = psutil.virtual_memory()- if mem.available / mem.total < free_mem_ratio:- return True- return False-- return [ResourceLimit()]---class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):- """The class for tuning XGBoost with unlimited depth, using sklearn API."""-- DEFAULT_ITER = 10-- @classmethod- def search_space(cls, data_size, **params):- space = XGBoostEstimator.search_space(data_size)- space.pop("max_depth")- return space-- @classmethod- def cost_relative2lgbm(cls):- return XGBoostEstimator.cost_relative2lgbm()-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- max_depth = params["max_depth"] = params.get("max_depth", 0)- if max_depth == 0:- params["grow_policy"] = params.get("grow_policy", "lossguide")- params["tree_method"] = params.get("tree_method", "hist")- params["use_label_encoder"] = params.get("use_label_encoder", False)- return params-- def __init__(- self,- task="binary",- **config,- ):- super().__init__(task, **config)- del self.params["verbose"]- self.params["verbosity"] = 0- import xgboost as xgb-- self.estimator_class = xgb.XGBRegressor- if "rank" == task:- self.estimator_class = xgb.XGBRanker- elif task in CLASSIFICATION:- self.estimator_class = xgb.XGBClassifier- self._xgb_version = xgb.__version__-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- if issparse(X_train) and self._xgb_version < "1.6.0":- # "auto" fails for sparse input since xgboost 1.6.0- self.params["tree_method"] = "auto"- if kwargs.get("gpu_per_trial"):- self.params["tree_method"] = "gpu_hist"- kwargs.pop("gpu_per_trial")- return super().fit(X_train, y_train, budget, free_mem_ratio, **kwargs)-- def _callbacks(self, start_time, deadline, free_mem_ratio) -> List[Callable]:- return XGBoostEstimator._callbacks(start_time, deadline, free_mem_ratio)---class XGBoostLimitDepthEstimator(XGBoostSklearnEstimator):- """The class for tuning XGBoost with limited depth, using sklearn API."""-- @classmethod- def search_space(cls, data_size, **params):- space = XGBoostEstimator.search_space(data_size)- space.pop("max_leaves")- upper = max(6, int(np.log2(data_size[0])))- space["max_depth"] = {- "domain": tune.randint(lower=1, upper=min(upper, 16)),- "init_value": 6,- "low_cost_init_value": 1,- }- space["learning_rate"]["init_value"] = 0.3- space["n_estimators"]["init_value"] = 10- return space-- @classmethod- def cost_relative2lgbm(cls):- return 64---class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):- """The class for tuning Random Forest."""-- HAS_CALLBACK = False- nrows = 101-- @classmethod- def search_space(cls, data_size, task, **params):- RandomForestEstimator.nrows = int(data_size[0])- upper = min(2048, RandomForestEstimator.nrows)- init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1- lower = min(0.1, init)- space = {- "n_estimators": {- "domain": tune.lograndint(lower=4, upper=max(5, upper)),- "init_value": 4,- "low_cost_init_value": 4,- },- "max_features": {- "domain": tune.loguniform(lower=lower, upper=1.0),- "init_value": init,- },- "max_leaves": {- "domain": tune.lograndint(- lower=4,- upper=max(5, min(32768, RandomForestEstimator.nrows >> 1)), #- ),- "init_value": 4,- "low_cost_init_value": 4,- },- }- if task in CLASSIFICATION:- space["criterion"] = {- "domain": tune.choice(["gini", "entropy"]),- # "init_value": "gini",- }- return space-- @classmethod- def cost_relative2lgbm(cls):- return 2-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- if "max_leaves" in params:- params["max_leaf_nodes"] = params.get(- "max_leaf_nodes", params.pop("max_leaves")- )- if self._task not in CLASSIFICATION and "criterion" in config:- params.pop("criterion")- if "random_state" not in params:- params["random_state"] = 12032022- return params-- def __init__(- self,- task="binary",- **params,- ):- super().__init__(task, **params)- self.params["verbose"] = 0- self.estimator_class = RandomForestRegressor- if task in CLASSIFICATION:- self.estimator_class = RandomForestClassifier---class ExtraTreesEstimator(RandomForestEstimator):- """The class for tuning Extra Trees."""-- @classmethod- def cost_relative2lgbm(cls):- return 1.9-- def __init__(self, task="binary", **params):- super().__init__(task, **params)- if "regression" in task:- self.estimator_class = ExtraTreesRegressor- else:- self.estimator_class = ExtraTreesClassifier---class LRL1Classifier(SKLearnEstimator):- """The class for tuning Logistic Regression with L1 regularization."""-- @classmethod- def search_space(cls, **params):- return {- "C": {- "domain": tune.loguniform(lower=0.03125, upper=32768.0),- "init_value": 1.0,- },- }-- @classmethod- def cost_relative2lgbm(cls):- return 160-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- params["tol"] = params.get("tol", 0.0001)- params["solver"] = params.get("solver", "saga")- params["penalty"] = params.get("penalty", "l1")- return params-- def __init__(self, task="binary", **config):- super().__init__(task, **config)- assert task in CLASSIFICATION, "LogisticRegression for classification task only"- self.estimator_class = LogisticRegression---class LRL2Classifier(SKLearnEstimator):- """The class for tuning Logistic Regression with L2 regularization."""-- limit_resource = True-- @classmethod- def search_space(cls, **params):- return LRL1Classifier.search_space(**params)-- @classmethod- def cost_relative2lgbm(cls):- return 25-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- params["tol"] = params.get("tol", 0.0001)- params["solver"] = params.get("solver", "lbfgs")- params["penalty"] = params.get("penalty", "l2")- return params-- def __init__(self, task="binary", **config):- super().__init__(task, **config)- assert task in CLASSIFICATION, "LogisticRegression for classification task only"- self.estimator_class = LogisticRegression---class CatBoostEstimator(BaseEstimator):- """The class for tuning CatBoost."""-- ITER_HP = "n_estimators"- DEFAULT_ITER = 1000-- @classmethod- def search_space(cls, data_size, **params):- upper = max(min(round(1500000 / data_size[0]), 150), 12)- return {- "early_stopping_rounds": {- "domain": tune.lograndint(lower=10, upper=upper),- "init_value": 10,- "low_cost_init_value": 10,- },- "learning_rate": {- "domain": tune.loguniform(lower=0.005, upper=0.2),- "init_value": 0.1,- },- "n_estimators": {- "domain": 8192,- "init_value": 8192,- },- }-- @classmethod- def size(cls, config):- n_estimators = config.get("n_estimators", 8192)- max_leaves = 64- return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8-- @classmethod- def cost_relative2lgbm(cls):- return 15-- def _preprocess(self, X):- if isinstance(X, DataFrame):- cat_columns = X.select_dtypes(include=["category"]).columns- if not cat_columns.empty:- X = X.copy()- X[cat_columns] = X[cat_columns].apply(- lambda x: x.cat.rename_categories(- [- str(c) if isinstance(c, float) else c- for c in x.cat.categories- ]- )- )- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":- # numpy array is not of numeric dtype- X = DataFrame(X)- for col in X.columns:- if isinstance(X[col][0], str):- X[col] = X[col].astype("category").cat.codes- X = X.to_numpy()- return X-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- params["n_estimators"] = params.get("n_estimators", 8192)- if "n_jobs" in params:- params["thread_count"] = params.pop("n_jobs")- return params-- def __init__(- self,- task="binary",- **config,- ):- super().__init__(task, **config)- self.params.update(- {- "verbose": config.get("verbose", False),- "random_seed": config.get("random_seed", 10242048),- }- )- from catboost import CatBoostRegressor-- self.estimator_class = CatBoostRegressor- if task in CLASSIFICATION:- from catboost import CatBoostClassifier-- self.estimator_class = CatBoostClassifier-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- start_time = time.time()- deadline = start_time + budget if budget else np.inf- train_dir = f"catboost_{str(start_time)}"- X_train = self._preprocess(X_train)- if isinstance(X_train, DataFrame):- cat_features = list(X_train.select_dtypes(include="category").columns)- else:- cat_features = []- use_best_model = kwargs.get("use_best_model", True)- n = (- max(int(len(y_train) * 0.9), len(y_train) - 1000)- if use_best_model- else len(y_train)- )- X_tr, y_tr = X_train[:n], y_train[:n]- from catboost import Pool, __version__-- eval_set = (- Pool(data=X_train[n:], label=y_train[n:], cat_features=cat_features)- if use_best_model- else None- )- if "sample_weight" in kwargs:- weight = kwargs["sample_weight"]- if weight is not None:- kwargs["sample_weight"] = weight[:n]- else:- weight = None-- model = self.estimator_class(train_dir=train_dir, **self.params)- if __version__ >= "0.26":- model.fit(- X_tr,- y_tr,- cat_features=cat_features,- eval_set=eval_set,- callbacks=CatBoostEstimator._callbacks(- start_time, deadline, free_mem_ratio if use_best_model else None- ),- **kwargs,- )- else:- model.fit(- X_tr,- y_tr,- cat_features=cat_features,- eval_set=eval_set,- **kwargs,- )- shutil.rmtree(train_dir, ignore_errors=True)- if weight is not None:- kwargs["sample_weight"] = weight- self._model = model- self.params[self.ITER_HP] = self._model.tree_count_- train_time = time.time() - start_time- return train_time-- @classmethod- def _callbacks(cls, start_time, deadline, free_mem_ratio):- class ResourceLimit:- def after_iteration(self, info) -> bool:- now = time.time()- if info.iteration == 1:- self._time_per_iter = now - start_time- if now + self._time_per_iter > deadline:- return False- if psutil is not None and free_mem_ratio is not None:- mem = psutil.virtual_memory()- if mem.available / mem.total < free_mem_ratio:- return False- return True # can continue-- return [ResourceLimit()]---class KNeighborsEstimator(BaseEstimator):- @classmethod- def search_space(cls, data_size, **params):- upper = min(512, int(data_size[0] / 2))- return {- "n_neighbors": {- "domain": tune.lograndint(lower=1, upper=max(2, upper)),- "init_value": 5,- "low_cost_init_value": 1,- },- }-- @classmethod- def cost_relative2lgbm(cls):- return 30-- def config2params(self, config: dict) -> dict:- params = super().config2params(config)- params["weights"] = params.get("weights", "distance")- return params-- def __init__(self, task="binary", **config):- super().__init__(task, **config)- if task in CLASSIFICATION:- from sklearn.neighbors import KNeighborsClassifier-- self.estimator_class = KNeighborsClassifier- else:- from sklearn.neighbors import KNeighborsRegressor-- self.estimator_class = KNeighborsRegressor-- def _preprocess(self, X):- if isinstance(X, DataFrame):- cat_columns = X.select_dtypes(["category"]).columns- if X.shape[1] == len(cat_columns):- raise ValueError("kneighbor requires at least one numeric feature")- X = X.drop(cat_columns, axis=1)- elif isinstance(X, np.ndarray) and X.dtype.kind not in "buif":- # drop categocial columns if any- X = DataFrame(X)- cat_columns = []- for col in X.columns:- if isinstance(X[col][0], str):- cat_columns.append(col)- X = X.drop(cat_columns, axis=1)- X = X.to_numpy()- return X---class Prophet(SKLearnEstimator):- """The class for tuning Prophet."""-- @classmethod- def search_space(cls, **params):- space = {- "changepoint_prior_scale": {- "domain": tune.loguniform(lower=0.001, upper=0.05),- "init_value": 0.05,- "low_cost_init_value": 0.001,- },- "seasonality_prior_scale": {- "domain": tune.loguniform(lower=0.01, upper=10),- "init_value": 10,- },- "holidays_prior_scale": {- "domain": tune.loguniform(lower=0.01, upper=10),- "init_value": 10,- },- "seasonality_mode": {- "domain": tune.choice(["additive", "multiplicative"]),- "init_value": "multiplicative",- },- }- return space-- def __init__(self, task="ts_forecast", n_jobs=1, **params):- super().__init__(task, **params)-- def _join(self, X_train, y_train):- assert TS_TIMESTAMP_COL in X_train, (- "Dataframe for training ts_forecast model must have column"- f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'- )- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])- train_df = X_train.join(y_train)- return train_df-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- from prophet import Prophet-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- cols = list(train_df)- cols.remove(TS_TIMESTAMP_COL)- cols.remove(TS_VALUE_COL)- logging.getLogger("prophet").setLevel(logging.WARNING)- model = Prophet(**self.params)- for regressor in cols:- model.add_regressor(regressor)- with suppress_stdout_stderr():- model.fit(train_df)- train_time = time.time() - current_time- self._model = model- return train_time-- def predict(self, X, **kwargs):- if isinstance(X, int):- raise ValueError(- "predict() with steps is only supported for arima/sarimax."- " For Prophet, pass a dataframe with the first column containing"- " the timestamp values."- )- if self._model is not None:- X = self._preprocess(X)- forecast = self._model.predict(X, **kwargs)- return forecast["yhat"]- else:- logger.warning(- "Estimator is not fit yet. Please run fit() before predict()."- )- return np.ones(X.shape[0])-- def score(self, X_val: DataFrame, y_val: Series, **kwargs):- from sklearn.metrics import r2_score- from .ml import metric_loss_score-- y_pred = self.predict(X_val, **kwargs)- self._metric = kwargs.get("metric", None)- if self._metric:- return metric_loss_score(self._metric, y_pred, y_val)- else:- return r2_score(y_pred, y_val)---class ARIMA(Prophet):- """The class for tuning ARIMA."""-- @classmethod- def search_space(cls, **params):- space = {- "p": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "d": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- }- return space-- def _join(self, X_train, y_train):- train_df = super()._join(X_train, y_train)- train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])- train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)- return train_df-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings-- warnings.filterwarnings("ignore")- from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- regressors = list(train_df)- regressors.remove(TS_VALUE_COL)- if regressors:- model = ARIMA_estimator(- train_df[[TS_VALUE_COL]],- exog=train_df[regressors],- order=(self.params["p"], self.params["d"], self.params["q"]),- enforce_stationarity=False,- enforce_invertibility=False,- )- else:- model = ARIMA_estimator(- train_df,- order=(self.params["p"], self.params["d"], self.params["q"]),- enforce_stationarity=False,- enforce_invertibility=False,- )- with suppress_stdout_stderr():- model = model.fit()- train_time = time.time() - current_time- self._model = model- return train_time-- def predict(self, X, **kwargs):- if self._model is not None:- if isinstance(X, int):- forecast = self._model.forecast(steps=X)- elif isinstance(X, DataFrame):- start = X[TS_TIMESTAMP_COL].iloc[0]- end = X[TS_TIMESTAMP_COL].iloc[-1]- if len(X.columns) > 1:- X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))- regressors = list(X)- forecast = self._model.predict(- start=start, end=end, exog=X[regressors], **kwargs- )- else:- forecast = self._model.predict(start=start, end=end, **kwargs)- else:- raise ValueError(- "X needs to be either a pandas Dataframe with dates as the first column"- " or an int number of periods for predict()."- )- return forecast- else:- return np.ones(X if isinstance(X, int) else X.shape[0])---class SARIMAX(ARIMA):- """The class for tuning SARIMA."""-- @classmethod- def search_space(cls, **params):- space = {- "p": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "d": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "P": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "D": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "Q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "s": {- "domain": tune.choice([1, 4, 6, 12]),- "init_value": 12,- },- }- return space-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings-- warnings.filterwarnings("ignore")- from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- regressors = list(train_df)- regressors.remove(TS_VALUE_COL)- if regressors:- model = SARIMAX_estimator(- train_df[[TS_VALUE_COL]],- exog=train_df[regressors],- order=(self.params["p"], self.params["d"], self.params["q"]),- seasonal_order=(- self.params["P"],- self.params["D"],- self.params["Q"],- self.params["s"],- ),- enforce_stationarity=False,- enforce_invertibility=False,- )- else:- model = SARIMAX_estimator(- train_df,- order=(self.params["p"], self.params["d"], self.params["q"]),- seasonal_order=(- self.params["P"],- self.params["D"],- self.params["Q"],- self.params["s"],- ),- enforce_stationarity=False,- enforce_invertibility=False,- )- with suppress_stdout_stderr():- model = model.fit()- train_time = time.time() - current_time- self._model = model- return train_time---class TS_SKLearn(SKLearnEstimator):- """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""-- base_class = SKLearnEstimator-- @classmethod- def search_space(cls, data_size, pred_horizon, **params):- space = cls.base_class.search_space(data_size, **params)- space.update(- {- "optimize_for_horizon": {- "domain": tune.choice([True, False]),- "init_value": False,- "low_cost_init_value": False,- },- "lags": {- "domain": tune.randint(- lower=1, upper=max(2, int(np.sqrt(data_size[0])))- ),- "init_value": 3,- },- }- )- return space-- def __init__(self, task="ts_forecast", **params):- super().__init__(task, **params)- self.hcrystaball_model = None- self.ts_task = (- "regression" if task in TS_FORECASTREGRESSION else "classification"- )-- def transform_X(self, X):- cols = list(X)- if len(cols) == 1:- ds_col = cols[0]- X = DataFrame(index=X[ds_col])- elif len(cols) > 1:- ds_col = cols[0]- exog_cols = cols[1:]- X = X[exog_cols].set_index(X[ds_col])- return X-- def _fit(self, X_train, y_train, budget=None, **kwargs):- from hcrystalball.wrappers import get_sklearn_wrapper-- X_train = self.transform_X(X_train)- X_train = self._preprocess(X_train)- params = self.params.copy()- lags = params.pop("lags")- optimize_for_horizon = params.pop("optimize_for_horizon")- estimator = self.base_class(task=self.ts_task, **params)- self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)- self.hcrystaball_model.lags = int(lags)- self.hcrystaball_model.fit(X_train, y_train)- if optimize_for_horizon:- # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon- model_list = []- for i in range(1, kwargs["period"] + 1):- (- X_fit,- y_fit,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(- X_train, y_train, i- )- self.hcrystaball_model.model.set_params(**estimator.params)- model = self.hcrystaball_model.model.fit(X_fit, y_fit)- model_list.append(model)- self._model = model_list- else:- (- X_fit,- y_fit,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(- X_train, y_train, kwargs["period"]- )- self.hcrystaball_model.model.set_params(**estimator.params)- model = self.hcrystaball_model.model.fit(X_fit, y_fit)- self._model = model-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- current_time = time.time()- self._fit(X_train, y_train, budget=budget, **kwargs)- train_time = time.time() - current_time- return train_time-- def predict(self, X, **kwargs):- if self._model is not None:- X = self.transform_X(X)- X = self._preprocess(X)- if isinstance(self._model, list):- assert len(self._model) == len(- X- ), "Model is optimized for horizon, length of X must be equal to `period`."- preds = []- for i in range(1, len(self._model) + 1):- (- X_pred,- _,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(- X.iloc[:i, :]- )- preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])- forecast = DataFrame(- data=np.asarray(preds).reshape(-1, 1),- columns=[self.hcrystaball_model.name],- index=X.index,- )- else:- (- X_pred,- _,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)- forecast = self._model.predict(X_pred, **kwargs)- return forecast- else:- logger.warning(- "Estimator is not fit yet. Please run fit() before predict()."- )- return np.ones(X.shape[0])---class LGBM_TS(TS_SKLearn):- """The class for tuning LGBM Regressor for time-series forecasting"""-- base_class = LGBMEstimator---class XGBoost_TS(TS_SKLearn):- """The class for tuning XGBoost Regressor for time-series forecasting"""-- base_class = XGBoostSklearnEstimator---# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball-# class CatBoost_TS_Regressor(TS_Regressor):-# base_class = CatBoostEstimator---class RF_TS(TS_SKLearn):- """The class for tuning Random Forest Regressor for time-series forecasting"""-- base_class = RandomForestEstimator---class ExtraTrees_TS(TS_SKLearn):- """The class for tuning Extra Trees Regressor for time-series forecasting"""-- base_class = ExtraTreesEstimator---class XGBoostLimitDepth_TS(TS_SKLearn):- """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""-- base_class = XGBoostLimitDepthEstimator---class TemporalFusionTransformerEstimator(SKLearnEstimator):- """The class for tuning Temporal Fusion Transformer"""-- @classmethod- def search_space(cls, data_size, pred_horizon, **params):- space = {- "gradient_clip_val": {- "domain": tune.loguniform(lower=0.01, upper=100.0),- "init_value": 0.01,- },- "hidden_size": {- "domain": tune.lograndint(lower=8, upper=512),- "init_value": 16,- },- "hidden_continuous_size": {- "domain": tune.randint(lower=1, upper=65),- "init_value": 8,- },- "attention_head_size": {- "domain": tune.randint(lower=1, upper=5),- "init_value": 4,- },- "dropout": {- "domain": tune.uniform(lower=0.1, upper=0.3),- "init_value": 0.1,- },- "learning_rate": {- "domain": tune.loguniform(lower=0.00001, upper=1.0),- "init_value": 0.001,- },- }- return space-- def transform_ds(self, X_train, y_train, **kwargs):- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])- self.data = X_train.join(y_train)-- max_prediction_length = kwargs["period"]- self.max_encoder_length = kwargs["max_encoder_length"]- training_cutoff = self.data["time_idx"].max() - max_prediction_length-- from pytorch_forecasting import TimeSeriesDataSet- from pytorch_forecasting.data import GroupNormalizer-- self.group_ids = kwargs["group_ids"].copy()- training = TimeSeriesDataSet(- self.data[lambda x: x.time_idx <= training_cutoff],- time_idx="time_idx",- target=TS_VALUE_COL,- group_ids=self.group_ids,- min_encoder_length=kwargs.get(- "min_encoder_length", self.max_encoder_length // 2- ), # keep encoder length long (as it is in the validation set)- max_encoder_length=self.max_encoder_length,- min_prediction_length=1,- max_prediction_length=max_prediction_length,- static_categoricals=kwargs.get("static_categoricals", []),- static_reals=kwargs.get("static_reals", []),- time_varying_known_categoricals=kwargs.get(- "time_varying_known_categoricals", []- ),- time_varying_known_reals=kwargs.get("time_varying_known_reals", []),- time_varying_unknown_categoricals=kwargs.get(- "time_varying_unknown_categoricals", []- ),- time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),- variable_groups=kwargs.get(- "variable_groups", {}- ), # group of categorical variables can be treated as one variable- lags=kwargs.get("lags", {}),- target_normalizer=GroupNormalizer(- groups=kwargs["group_ids"], transformation="softplus"- ), # use softplus and normalize by group- add_relative_time_idx=True,- add_target_scales=True,- add_encoder_length=True,- )-- # create validation set (predict=True) which means to predict the last max_prediction_length points in time- # for each series- validation = TimeSeriesDataSet.from_dataset(- training, self.data, predict=True, stop_randomization=True- )-- # create dataloaders for model- batch_size = kwargs.get("batch_size", 64)- train_dataloader = training.to_dataloader(- train=True, batch_size=batch_size, num_workers=0- )- val_dataloader = validation.to_dataloader(- train=False, batch_size=batch_size * 10, num_workers=0- )-- return training, train_dataloader, val_dataloader-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings- import pytorch_lightning as pl- from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor- import torch- from pytorch_forecasting import TemporalFusionTransformer- from pytorch_forecasting.metrics import QuantileLoss-- warnings.filterwarnings("ignore")- current_time = time.time()- training, train_dataloader, val_dataloader = self.transform_ds(- X_train, y_train, **kwargs- )- params = self.params.copy()- gradient_clip_val = params.pop("gradient_clip_val")- params.pop("n_jobs")- max_epochs = kwargs.get("max_epochs", 20)- early_stop_callback = EarlyStopping(- monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min"- )-- def _fit(log):- default_trainer_kwargs = dict(- gpus=kwargs.get("gpu_per_trial", [0])- if torch.cuda.is_available()- else None,- max_epochs=max_epochs,- gradient_clip_val=gradient_clip_val,- callbacks=[LearningRateMonitor(), early_stop_callback]- if log- else [early_stop_callback],- logger=log,- )- trainer = pl.Trainer(- **default_trainer_kwargs,- )- tft = TemporalFusionTransformer.from_dataset(- training,- **params,- lstm_layers=2, # 2 is mostly optimal according to documentation- output_size=7, # 7 quantiles by default- loss=QuantileLoss(),- log_interval=10 if log else 0,- # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches- reduce_on_plateau_patience=4,- )- # fit network- trainer.fit(- tft,- train_dataloaders=train_dataloader,- val_dataloaders=val_dataloader,- )- return trainer-- # try:- # from pytorch_lightning.loggers import TensorBoardLogger-- # logger = TensorBoardLogger(- # kwargs.get("log_dir", "lightning_logs")- # ) # logging results to a tensorboard- # trainer = _fit(log=logger)- # except ValueError:- # issue with pytorch forecasting model log_prediction() function- # pytorch-forecasting issue #1145- trainer = _fit(log=False)- best_model_path = trainer.checkpoint_callback.best_model_path- best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)- train_time = time.time() - current_time- self._model = best_tft- return train_time-- def predict(self, X):- import pandas as pd-- ids = self.group_ids.copy()- ids.append(TS_TIMESTAMP_COL)- encoder_data = self.data[- lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length- ]- # following pytorchforecasting example, make all target values equal to the last data- last_data_cols = self.group_ids.copy()- last_data_cols.append(TS_VALUE_COL)- last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]- decoder_data = X- if "time_idx" not in decoder_data:- decoder_data = add_time_idx_col(decoder_data)- decoder_data["time_idx"] += (- encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()- )- # decoder_data[TS_VALUE_COL] = 0- decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)- decoder_data = decoder_data.sort_values(ids)- new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)- new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")- new_raw_predictions = self._model.predict(new_prediction_data)- index = [decoder_data[idx].to_numpy() for idx in ids]- predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)- return predictions---class suppress_stdout_stderr(object):- def __init__(self):- # Open a pair of null files- self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]- # Save the actual stdout (1) and stderr (2) file descriptors.- self.save_fds = (os.dup(1), os.dup(2))-- def __enter__(self):- # Assign the null pointers to stdout and stderr.- os.dup2(self.null_fds[0], 1)- os.dup2(self.null_fds[1], 2)-- def __exit__(self, *_):- # Re-assign the real stdout/stderr back to (1) and (2)- os.dup2(self.save_fds[0], 1)- os.dup2(self.save_fds[1], 2)- # Close the null files- os.close(self.null_fds[0])- os.close(self.null_fds[1])+import warnings++from flaml.automl.model import *+++warnings.warn(+ "Importing from `flaml.model` is deprecated. Please use `flaml.automl.model`.",+ DeprecationWarning,+)diff --git a/test/automl/test_classification.py b/test/automl/test_classification.pyindex 19726029e..adb308166 100644--- a/test/automl/test_classification.py+++ b/test/automl/test_classification.py@@ -6,7 +6,7 @@ from sklearn.model_selection import train_test_splitimport pandas as pdfrom datetime import datetimefrom flaml import AutoML-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimatorfrom flaml import tunediff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.pyindex 34ffb2ec5..4b9c040e4 100644--- a/test/automl/test_forecast.py+++ b/test/automl/test_forecast.py@@ -58,12 +58,12 @@ def test_forecast_automl(budget=5):print("Predicted labels", y_pred)print("True labels", y_test)""" compute different metric values on testing dataset"""- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoremape = sklearn_metric_loss_score("mape", y_pred, y_test)print("mape", "=", mape)assert mape <= 0.005, "the mape of flaml should be less than 0.005"- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,@@ -212,10 +212,10 @@ def test_multivariate_forecast_num(budget=5):print("Predicted labels", y_pred)print("True labels", y_test)""" compute different metric values on testing dataset"""- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoreprint("mape", "=", sklearn_metric_loss_score("mape", y_pred, y_test))- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,@@ -339,13 +339,13 @@ def test_multivariate_forecast_cat(budget=5):print("Predicted labels", y_pred)print("True labels", y_test)""" compute different metric values on testing dataset"""- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoreprint("mape", "=", sklearn_metric_loss_score("mape", y_pred, y_test))print("rmse", "=", sklearn_metric_loss_score("rmse", y_pred, y_test))print("mse", "=", sklearn_metric_loss_score("mse", y_pred, y_test))print("mae", "=", sklearn_metric_loss_score("mae", y_pred, y_test))- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,@@ -413,12 +413,12 @@ def test_forecast_classification(budget=5):""" compute predictions of testing dataset """y_pred = automl.predict(X_test)""" compute different metric values on testing dataset"""- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoreprint(y_test)print(y_pred)print("accuracy", "=", 1 - sklearn_metric_loss_score("accuracy", y_pred, y_test))- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,@@ -554,7 +554,7 @@ def test_forecast_panel(budget=5):""" compute predictions of testing dataset """y_pred = automl.predict(X_test)""" compute different metric values on testing dataset"""- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoreprint(y_test)print(y_pred)@@ -575,7 +575,7 @@ def test_forecast_panel(budget=5):# """compute prediction for a specific time series"""# a01_sku01_preds = automl.predict(X_test[(X_test["agency"] == "Agency_01") & (X_test["sku"] == "SKU_01")])# print("Agency01 SKU_01 predictions: ", a01_sku01_preds)- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,diff --git a/test/automl/test_multiclass.py b/test/automl/test_multiclass.pyindex a0f18bbcb..f6bb38396 100644--- a/test/automl/test_multiclass.py+++ b/test/automl/test_multiclass.py@@ -3,10 +3,10 @@ import numpy as npimport scipy.sparsefrom sklearn.datasets import load_iris, load_winefrom flaml import AutoML-from flaml.data import CLASSIFICATION, get_output_from_log-from flaml.model import LGBMEstimator, XGBoostSklearnEstimator, SKLearnEstimator+from flaml.automl.data import CLASSIFICATION, get_output_from_log+from flaml.automl.model import LGBMEstimator, XGBoostSklearnEstimator, SKLearnEstimatorfrom flaml import tune-from flaml.training_log import training_log_reader+from flaml.automl.training_log import training_log_readerclass MyRegularizedGreedyForest(SKLearnEstimator):@@ -280,7 +280,7 @@ class TestMultiClass(unittest.TestCase):estimator = automl_experiment_macro.modely_pred = estimator.predict(X_train)y_pred_proba = estimator.predict_proba(X_train)- from flaml.ml import norm_confusion_matrix, multi_class_curves+ from flaml.automl.ml import norm_confusion_matrix, multi_class_curvesprint(norm_confusion_matrix(y_train, y_pred))from sklearn.metrics import roc_curve, precision_recall_curvediff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.pyindex 09be23ebb..be86b3b5b 100644--- a/test/automl/test_notebook_example.py+++ b/test/automl/test_notebook_example.py@@ -4,7 +4,7 @@ from requests.exceptions import ChunkedEncodingError, SSLErrordef test_automl(budget=5, dataset_format="dataframe", hpo_method=None):- from flaml.data import load_openml_dataset+ from flaml.automl.data import load_openml_datasetimport urllib3performance_check_budget = 600@@ -77,7 +77,7 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):print("True labels", y_test)y_pred_proba = automl.predict_proba(X_test)[:, 1]""" compute different metric values on testing dataset """- from flaml.ml import sklearn_metric_loss_score+ from flaml.automl.ml import sklearn_metric_loss_scoreaccuracy = 1 - sklearn_metric_loss_score("accuracy", y_pred, y_test)print("accuracy", "=", accuracy)@@ -87,7 +87,7 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):print("log_loss", "=", sklearn_metric_loss_score("log_loss", y_pred_proba, y_test))if budget is None:assert accuracy >= 0.669, "the accuracy of flaml should be larger than 0.67"- from flaml.data import get_output_from_log+ from flaml.automl.data import get_output_from_log(time_history,@@ -119,7 +119,7 @@ def _test_nobudget():def test_mlflow():# subprocess.check_call([sys.executable, "-m", "pip", "install", "mlflow"])import mlflow- from flaml.data import load_openml_task+ from flaml.automl.data import load_openml_tasktry:X_train, X_test, y_train, y_test = load_openml_task(diff --git a/test/automl/test_regression.py b/test/automl/test_regression.pyindex 0aca92f15..02d058c67 100644--- a/test/automl/test_regression.py+++ b/test/automl/test_regression.py@@ -6,8 +6,8 @@ from sklearn.datasets import ()from flaml import AutoML-from flaml.data import get_output_from_log-from flaml.model import XGBoostEstimator+from flaml.automl.data import get_output_from_log+from flaml.automl.model import XGBoostEstimatordef logregobj(preds, dtrain):diff --git a/test/automl/test_training_log.py b/test/automl/test_training_log.pyindex 85cccbe34..ff1b42673 100644--- a/test/automl/test_training_log.py+++ b/test/automl/test_training_log.py@@ -5,7 +5,7 @@ from tempfile import TemporaryDirectoryfrom sklearn.datasets import fetch_california_housingfrom flaml import AutoML-from flaml.training_log import training_log_reader+from flaml.automl.training_log import training_log_readerclass TestTrainingLog(unittest.TestCase):diff --git a/test/automl/test_warmstart.py b/test/automl/test_warmstart.pyindex c443cf6ce..74cf8c37f 100644--- a/test/automl/test_warmstart.py+++ b/test/automl/test_warmstart.py@@ -2,7 +2,7 @@ import unittestimport numpy as npfrom sklearn.datasets import load_irisfrom flaml import AutoML-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimatorfrom flaml import tune@@ -124,7 +124,7 @@ class TestWarmStart(unittest.TestCase):print(automl.best_config_per_estimator)def test_FLAML_sample_size_in_starting_points(self):- from flaml.data import load_openml_dataset+ from flaml.automl.data import load_openml_datasetfrom flaml import AutoMLX_train, X_test, y_train, y_test = load_openml_dataset(diff --git a/test/automl/test_xgboost2d.py b/test/automl/test_xgboost2d.pyindex 7c2eba1c2..46150a257 100644--- a/test/automl/test_xgboost2d.py+++ b/test/automl/test_xgboost2d.py@@ -3,7 +3,7 @@ import unittestfrom sklearn.datasets import fetch_openmlfrom sklearn.model_selection import train_test_splitfrom flaml.automl import AutoML-from flaml.model import XGBoostSklearnEstimator+from flaml.automl.model import XGBoostSklearnEstimatorfrom flaml import tunediff --git a/test/automl/test_xgboost2d_sample_size.py b/test/automl/test_xgboost2d_sample_size.pyindex bdc7f8c1e..aabba9139 100644--- a/test/automl/test_xgboost2d_sample_size.py+++ b/test/automl/test_xgboost2d_sample_size.py@@ -3,7 +3,7 @@ import unittestfrom sklearn.datasets import fetch_openmlfrom sklearn.model_selection import train_test_splitfrom flaml.automl import AutoML-from flaml.model import XGBoostSklearnEstimator+from flaml.automl.model import XGBoostSklearnEstimatorfrom flaml import tunediff --git a/test/default/test_defaults.py b/test/default/test_defaults.pyindex 77408b695..22ef0a2d3 100644--- a/test/default/test_defaults.py+++ b/test/default/test_defaults.py@@ -5,12 +5,11 @@ from sklearn.model_selection import train_test_splitimport pandas as pdfrom flaml import AutoMLfrom flaml.default import (- portfolio,- regret,preprocess_and_suggest_hyperparams,suggest_hyperparams,suggest_learner,)+from flaml.default import portfolio, regretdef test_greedy_feedback(path="test/default", strategy="greedy-feedback"):@@ -147,7 +146,7 @@ def test_suggest_regression():def test_rf():- from flaml.default.estimator import RandomForestRegressor, RandomForestClassifier+ from flaml.default import RandomForestRegressor, RandomForestClassifierX_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)rf = RandomForestClassifier()@@ -165,7 +164,7 @@ def test_rf():def test_extratrees():- from flaml.default.estimator import ExtraTreesRegressor, ExtraTreesClassifier+ from flaml.default import ExtraTreesRegressor, ExtraTreesClassifierX_train, y_train = load_iris(return_X_y=True, as_frame=True)classifier = ExtraTreesClassifier()@@ -183,7 +182,7 @@ def test_extratrees():def test_lgbm():- from flaml.default.estimator import LGBMRegressor, LGBMClassifier+ from flaml.default import LGBMRegressor, LGBMClassifierX_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)classifier = LGBMClassifier(n_jobs=1)@@ -203,7 +202,7 @@ def test_lgbm():def test_xgboost():- from flaml.default.estimator import XGBRegressor, XGBClassifier+ from flaml.default import XGBRegressor, XGBClassifierX_train, y_train = load_breast_cancer(return_X_y=True, as_frame=True)classifier = XGBClassifier(max_depth=0)diff --git a/test/default_lgbm.py b/test/default_lgbm.pyindex 9085db489..105a3d6ec 100644--- a/test/default_lgbm.py+++ b/test/default_lgbm.py@@ -1,6 +1,6 @@-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml.default import LGBMRegressor-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_scoreX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")lgbm = LGBMRegressor()diff --git a/test/default_xgb.py b/test/default_xgb.pyindex bda7307e5..14a58deda 100644--- a/test/default_xgb.py+++ b/test/default_xgb.py@@ -1,6 +1,6 @@-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml.default import XGBClassifier-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_scoreX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")xgb = XGBClassifier()diff --git a/test/load_args.py b/test/load_args.pyindex 5c7c35378..9ffcba856 100644--- a/test/load_args.py+++ b/test/load_args.py@@ -1,5 +1,5 @@def test_load_args_sub():- from flaml.nlp.utils import TrainingArgumentsForAuto+ from flaml.automl.nlp.huggingface.training_args import TrainingArgumentsForAutoTrainingArgumentsForAuto.load_args_from_console()diff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.pyindex c653b74c5..0918e0d85 100644--- a/test/nlp/test_autohf_custom_metric.py+++ b/test/nlp/test_autohf_custom_metric.py@@ -19,7 +19,6 @@ def custom_metric(groups_train=None,):from datasets import Dataset- from flaml.model import TransformersEstimatorif estimator._trainer is None:trainer = estimator._init_model_for_predict()diff --git a/test/object_store.py b/test/object_store.pyindex 8539d1cb3..dc00914e6 100644--- a/test/object_store.py+++ b/test/object_store.py@@ -1,5 +1,5 @@from flaml import tune-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimatorimport lightgbmfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import fetch_california_housingdiff --git a/test/ray/distribute_tune.py b/test/ray/distribute_tune.pyindex 991082e8e..e78c624b4 100644--- a/test/ray/distribute_tune.py+++ b/test/ray/distribute_tune.py@@ -5,7 +5,7 @@ from sklearn.datasets import load_breast_cancerfrom sklearn.metrics import accuracy_scorefrom sklearn.model_selection import train_test_splitfrom flaml import tune-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimatordef train_breast_cancer(config):diff --git a/test/rep.py b/test/rep.pyindex e78980e78..266ac0059 100644--- a/test/rep.py+++ b/test/rep.py@@ -1,5 +1,5 @@-from flaml.data import load_openml_dataset-from flaml.ml import ExtraTreesEstimator+from flaml.automl.data import load_openml_dataset+from flaml.automl.ml import ExtraTreesEstimatorfrom flaml import AutoMLX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")diff --git a/test/test_model.py b/test/test_model.pyindex 9a12b1b2e..b05f5d58a 100644--- a/test/test_model.py+++ b/test/test_model.py@@ -2,7 +2,7 @@ from sklearn.datasets import make_classificationimport numpy as npfrom pandas import DataFramefrom datetime import datetime-from flaml.model import (+from flaml.automl.model import (KNeighborsEstimator,LRL2Classifier,BaseEstimator,diff --git a/test/tune_example.py b/test/tune_example.pyindex a71300692..0dc770790 100644--- a/test/tune_example.py+++ b/test/tune_example.py@@ -1,5 +1,5 @@from flaml import tune-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimatorimport lightgbmfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import fetch_california_housingdiff --git a/website/docs/Examples/AutoML-for-LightGBM.md b/website/docs/Examples/AutoML-for-LightGBM.mdindex ad5c49ab9..0094dd561 100644--- a/website/docs/Examples/AutoML-for-LightGBM.md+++ b/website/docs/Examples/AutoML-for-LightGBM.md@@ -13,7 +13,7 @@ This option is not necessary in general.```pythonfrom flaml import AutoML-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_dataset# Download [houses dataset](https://www.openml.org/d/537) from OpenML. The task is to predict median price of the house in the region based on demographic composition and a state of housing market in the region.X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir='./')@@ -25,7 +25,7 @@ settings = {"estimator_list": ['lgbm'], # list of ML learners; we tune lightgbm in this example"task": 'regression', # task type"log_file_name": 'houses_experiment.log', # flaml log file- "seed": 7654321, # random seed+ "seed": 7654321, # random seed}automl.fit(X_train=X_train, y_train=y_train, **settings)```@@ -114,7 +114,8 @@ print('Predicted labels', y_pred)#### Compute different metric values on testing dataset```python-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_score+print('r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))print('mse', '=', sklearn_metric_loss_score('mse', y_pred, y_test))print('mae', '=', sklearn_metric_loss_score('mae', y_pred, y_test))@@ -131,7 +132,8 @@ from lightgbm import LGBMRegressorlgbm = LGBMRegressor()lgbm.fit(X_train, y_train)y_pred = lgbm.predict(X_test)-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_score+print('default lgbm r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))# default lgbm r2 = 0.8296179648694404```@@ -141,10 +143,10 @@ print('default lgbm r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_testHow does the model accuracy improve as we search for different hyperparameter configurations?```python-from flaml.data import get_output_from_log+from flaml.automl.data import get_output_from_logimport numpy as np-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =get_output_from_log(filename=settings['log_file_name'], time_budget=60)plt.title('Learning Curve')plt.xlabel('Wall Clock Time (s)')@@ -163,11 +165,12 @@ The native API of LightGBM allows one to specify a custom objective function in```pythonimport numpy as np+# define your customized objective functiondef my_loss_obj(y_true, y_pred):c = 0.5residual = y_pred - y_true- grad = c * residual /(np.abs(residual) + c)+ grad = c * residual / (np.abs(residual) + c)hess = c ** 2 / (np.abs(residual) + c) ** 2# rmse grad and hessgrad_rmse = residual@@ -180,10 +183,12 @@ def my_loss_obj(y_true, y_pred):hess_mae = 1.0coef = [0.4, 0.3, 0.3]- return coef[0] * grad + coef[1] * grad_rmse + coef[2] * grad_mae, \- coef[0] * hess + coef[1] * hess_rmse + coef[2] * hess_mae+ return coef[0] * grad + coef[1] * grad_rmse + coef[2] * grad_mae,+ coef[0] * hess + coef[1] * hess_rmse + coef[2] * hess_mae+++from flaml.automl.model import LGBMEstimator-from flaml.model import LGBMEstimatorclass MyLGBM(LGBMEstimator):"""LGBMEstimator with my_loss_obj as the objective function"""diff --git a/website/docs/Examples/AutoML-for-XGBoost.md b/website/docs/Examples/AutoML-for-XGBoost.mdindex 2b7bf6092..774050a53 100644--- a/website/docs/Examples/AutoML-for-XGBoost.md+++ b/website/docs/Examples/AutoML-for-XGBoost.md@@ -13,7 +13,7 @@ This option is not necessary in general.```pythonfrom flaml import AutoML-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_dataset# Download [houses dataset](https://www.openml.org/d/537) from OpenML. The task is to predict median price of the house in the region based on demographic composition and a state of housing market in the region.X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir='./')@@ -25,7 +25,7 @@ settings = {"estimator_list": ['xgboost'], # list of ML learners; we tune XGBoost in this example"task": 'regression', # task type"log_file_name": 'houses_experiment.log', # flaml log file- "seed": 7654321, # random seed+ "seed": 7654321, # random seed}automl.fit(X_train=X_train, y_train=y_train, **settings)```@@ -143,7 +143,8 @@ print('Predicted labels', y_pred)#### Compute different metric values on testing dataset```python-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_score+print('r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))print('mse', '=', sklearn_metric_loss_score('mse', y_pred, y_test))print('mae', '=', sklearn_metric_loss_score('mae', y_pred, y_test))@@ -160,7 +161,8 @@ from xgboost import XGBRegressorxgb = XGBRegressor()xgb.fit(X_train, y_train)y_pred = xgb.predict(X_test)-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_score+print('default xgboost r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))# default xgboost r2 = 0.8265451174596482```@@ -170,10 +172,10 @@ print('default xgboost r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_tHow does the model accuracy improve as we search for different hyperparameter configurations?```python-from flaml.data import get_output_from_log+from flaml.automl.data import get_output_from_logimport numpy as np-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =get_output_from_log(filename=settings['log_file_name'], time_budget=60)plt.title('Learning Curve')plt.xlabel('Wall Clock Time (s)')@@ -190,15 +192,18 @@ You can easily enable a custom objective function by adding a customized XGBoost```pythonimport numpy as np+# define your customized objective functiondef logregobj(preds, dtrain):labels = dtrain.get_label()- preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight+ preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weightgrad = preds - labelshess = preds * (1.0 - preds)return grad, hess-from flaml.model import XGBoostEstimator++from flaml.automl.model import XGBoostEstimator+class MyXGB1(XGBoostEstimator):'''XGBoostEstimator with the logregobj function as the objective function@@ -207,6 +212,7 @@ class MyXGB1(XGBoostEstimator):def __init__(self, **config):super().__init__(objective=logregobj, **config)+class MyXGB2(XGBoostEstimator):'''XGBoostEstimator with 'reg:squarederror' as the objective function'''diff --git a/website/docs/Examples/Default-Flamlized.md b/website/docs/Examples/Default-Flamlized.mdindex edc9b08b7..d87ee47f9 100644--- a/website/docs/Examples/Default-Flamlized.md+++ b/website/docs/Examples/Default-Flamlized.md@@ -9,9 +9,9 @@ This example requires openml==0.10.2.### Zero-shot AutoML```python-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml.default import LGBMRegressor-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_scoreX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")lgbm = LGBMRegressor()@@ -65,9 +65,9 @@ X_test.shape: (5160, 8), y_test.shape: (5160,)### Zero-shot AutoML```python-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml.default import XGBClassifier-from flaml.ml import sklearn_metric_loss_score+from flaml.automl.ml import sklearn_metric_loss_scoreX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")xgb = XGBClassifier()diff --git a/website/docs/Examples/Integrate - AzureML.md b/website/docs/Examples/Integrate - AzureML.mdindex 6a7e6a3c6..4d9db9088 100644--- a/website/docs/Examples/Integrate - AzureML.md+++ b/website/docs/Examples/Integrate - AzureML.md@@ -27,7 +27,7 @@ mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())### Start an AutoML run```python-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml import AutoML# Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure.diff --git a/website/docs/Examples/Integrate - Scikit-learn Pipeline.md b/website/docs/Examples/Integrate - Scikit-learn Pipeline.mdindex 7f272cea7..ee32ebd0d 100644--- a/website/docs/Examples/Integrate - Scikit-learn Pipeline.md+++ b/website/docs/Examples/Integrate - Scikit-learn Pipeline.md@@ -5,7 +5,7 @@ This example requires openml==0.10.2.### Load data```python-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_dataset# Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure.X_train, X_test, y_train, y_test = load_openml_dataset(diff --git a/website/docs/Getting-Started.md b/website/docs/Getting-Started.mdindex 0c54e2d45..e3e828cbc 100644--- a/website/docs/Getting-Started.md+++ b/website/docs/Getting-Started.md@@ -44,7 +44,8 @@ You can run generic hyperparameter tuning for a custom function (machine learnin```pythonfrom flaml import tune-from flaml.model import LGBMEstimator+from flaml.automl.model import LGBMEstimator+def train_lgbm(config: dict) -> dict:# convert config dict to lgbm params@@ -58,6 +59,7 @@ def train_lgbm(config: dict) -> dict:# return eval results as a dictionaryreturn {"mse": mse}+# load a built-in search space from flamlflaml_lgbm_search_space = LGBMEstimator.search_space(X_train.shape)# specify the search space as a dict from hp name to domain; you can define your own search space same way@@ -92,7 +94,7 @@ Then, you can use it just like you use the original `LGMBClassifier`. Your other* Find code examples under "Examples": from [AutoML - Classification](Examples/AutoML-Classification) to [Tune - PyTorch](Examples/Tune-PyTorch).* Find [talks](https://www.youtube.com/channel/UCfU0zfFXHXdAd5x-WvFBk5A) and [tutorials](https://github.com/microsoft/FLAML/tree/tutorial/tutorial) about FLAML.* Learn about [research](Research) around FLAML.-* Refer to [SDK](reference/automl) and [FAQ](FAQ).+* Refer to [SDK](reference/automl/automl) and [FAQ](FAQ).If you like our project, please give it a [star](https://github.com/microsoft/FLAML/stargazers) on GitHub. If you are interested in contributing, please read [Contributor's Guide](Contribute).diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.mdindex 7627a510d..94025e57e 100644--- a/website/docs/Use-Cases/Task-Oriented-AutoML.md+++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md@@ -2,7 +2,7 @@## Overview-[`flaml.AutoML`](../reference/automl#automl-objects) is a class for task-oriented AutoML. It can be used as a scikit-learn style estimator with the standard `fit` and `predict` functions. The minimal inputs from users are the training data and the task type.+[`flaml.AutoML`](../reference/automl/automl#automl-objects) is a class for task-oriented AutoML. It can be used as a scikit-learn style estimator with the standard `fit` and `predict` functions. The minimal inputs from users are the training data and the task type.* Training data:- numpy array. When the input data are stored in numpy array, they are passed to `fit()` as `X_train` and `y_train`.@@ -134,48 +134,49 @@ The estimator list can contain one or more estimator names, each corresponding t#### Guidelines on tuning a custom estimatorTo tune a custom estimator that is not built-in, you need to:-1. Build a custom estimator by inheritting [`flaml.model.BaseEstimator`](../reference/model#baseestimator-objects) or a derived class.+1. Build a custom estimator by inheritting [`flaml.model.BaseEstimator`](../reference/automl/model#baseestimator-objects) or a derived class.For example, if you have a estimator class with scikit-learn style `fit()` and `predict()` functions, you only need to set `self.estimator_class` to be that class in your constructor.```python-from flaml.model import SKLearnEstimator+from flaml.automl.model import SKLearnEstimator# SKLearnEstimator is derived from BaseEstimatorimport rgf-class MyRegularizedGreedyForest(SKLearnEstimator):- def __init__(self, task="binary", **config):- super().__init__(task, **config)-- if task in CLASSIFICATION:- from rgf.sklearn import RGFClassifier-- self.estimator_class = RGFClassifier- else:- from rgf.sklearn import RGFRegressor-- self.estimator_class = RGFRegressor- @classmethod- def search_space(cls, data_size, task):- space = {- "max_leaf": {- "domain": tune.lograndint(lower=4, upper=data_size),- "low_cost_init_value": 4,- },- "n_iter": {- "domain": tune.lograndint(lower=1, upper=data_size),- "low_cost_init_value": 1,- },- "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},- "min_samples_leaf": {- "domain": tune.lograndint(lower=1, upper=20),- "init_value": 20,- },- }- return space+class MyRegularizedGreedyForest(SKLearnEstimator):+ def __init__(self, task="binary", **config):+ super().__init__(task, **config)++ if task in CLASSIFICATION:+ from rgf.sklearn import RGFClassifier++ self.estimator_class = RGFClassifier+ else:+ from rgf.sklearn import RGFRegressor++ self.estimator_class = RGFRegressor++ @classmethod+ def search_space(cls, data_size, task):+ space = {+ "max_leaf": {+ "domain": tune.lograndint(lower=4, upper=data_size),+ "low_cost_init_value": 4,+ },+ "n_iter": {+ "domain": tune.lograndint(lower=1, upper=data_size),+ "low_cost_init_value": 1,+ },+ "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},+ "min_samples_leaf": {+ "domain": tune.lograndint(lower=1, upper=20),+ "init_value": 20,+ },+ }+ return space```-In the constructor, we set `self.estimator_class` as `RGFClassifier` or `RGFRegressor` according to the task type. If the estimator you want to tune does not have a scikit-learn style `fit()` and `predict()` API, you can override the `fit()` and `predict()` function of `flaml.model.BaseEstimator`, like [XGBoostEstimator](../reference/model#xgboostestimator-objects). Importantly, we also add the `task="binary"` parameter in the signature of `__init__` so that it doesn't get grouped together with the `**config` kwargs that determines the parameters with which the underlying estimator (`self.estimator_class`) is constructed. If your estimator doesn't use one of the parameters that it is passed, for example some regressors in `scikit-learn` don't use the `n_jobs` parameter, it is enough to add `n_jobs=None` to the signature so that it is ignored by the `**config` dict.+In the constructor, we set `self.estimator_class` as `RGFClassifier` or `RGFRegressor` according to the task type. If the estimator you want to tune does not have a scikit-learn style `fit()` and `predict()` API, you can override the `fit()` and `predict()` function of `flaml.model.BaseEstimator`, like [XGBoostEstimator](../reference/automl/model#xgboostestimator-objects). Importantly, we also add the `task="binary"` parameter in the signature of `__init__` so that it doesn't get grouped together with the `**config` kwargs that determines the parameters with which the underlying estimator (`self.estimator_class`) is constructed. If your estimator doesn't use one of the parameters that it is passed, for example some regressors in `scikit-learn` don't use the `n_jobs` parameter, it is enough to add `n_jobs=None` to the signature so that it is ignored by the `**config` dict.2. Give the custom estimator a name and add it in AutoML. E.g.,@@ -203,20 +204,22 @@ In the example above, we tune four hyperparameters, three integers and one floatTo customize the search space for a built-in estimator, use a similar approach to define a class that inherits the existing estimator. For example,```python-from flaml.model import XGBoostEstimator+from flaml.automl.model import XGBoostEstimator+def logregobj(preds, dtrain):- labels = dtrain.get_label()- preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight- grad = preds - labels- hess = preds * (1.0 - preds)- return grad, hess+ labels = dtrain.get_label()+ preds = 1.0 / (1.0 + np.exp(-preds)) # transform raw leaf weight+ grad = preds - labels+ hess = preds * (1.0 - preds)+ return grad, hess+class MyXGB1(XGBoostEstimator):- """XGBoostEstimator with logregobj as the objective function"""+ """XGBoostEstimator with logregobj as the objective function"""- def __init__(self, **config):- super().__init__(objective=logregobj, **config)+ def __init__(self, **config):+ super().__init__(objective=logregobj, **config)```We override the constructor and set the training objective as a custom function `logregobj`. The hyperparameters and their search range do not change. For another example,@@ -242,7 +245,7 @@ We override the `search_space` function to tune two hyperparameters only, "n_est##### A shortcut to override the search space-One can use the `custom_hp` argument in [`AutoML.fit()`](../reference/automl#fit) to override the search space for an existing estimator quickly. For example, if you would like to temporarily change the search range of "n_estimators" of xgboost, disable searching "max_leaves" in random forest, and add "subsample" in the search space of lightgbm, you can set:+One can use the `custom_hp` argument in [`AutoML.fit()`](../reference/automl/automl#fit) to override the search space for an existing estimator quickly. For example, if you would like to temporarily change the search range of "n_estimators" of xgboost, disable searching "max_leaves" in random forest, and add "subsample" in the search space of lightgbm, you can set:```pythoncustom_hp = {@@ -456,7 +459,7 @@ Extra fit arguments that are needed by the estimators can be passed to `AutoML.fIn addition, you can specify the different arguments needed by different estimators using the `fit_kwargs_by_estimator` argument. For example, you can set the custom arguments for a Transformers model:```python-from flaml.data import load_openml_dataset+from flaml.automl.data import load_openml_datasetfrom flaml import AutoMLX_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")@@ -487,7 +490,7 @@ print(automl.model)# <flaml.model.LGBMEstimator object at 0x7f9b502c4550>```-[`flaml.model.LGBMEstimator`](../reference/model#lgbmestimator-objects) is a wrapper class for LightGBM models. To access the underlying model, use the `estimator` property of the `flaml.model.LGBMEstimator` instance.+[`flaml.model.LGBMEstimator`](../reference/automl/model#lgbmestimator-objects) is a wrapper class for LightGBM models. To access the underlying model, use the `estimator` property of the `flaml.model.LGBMEstimator` instance.```pythonprint(automl.model.estimator)@@ -547,9 +550,9 @@ print(automl.config_history)To plot how the loss is improved over time during the model search, first load the search history from the log file:```python-from flaml.data import get_output_from_log+from flaml.automl.data import get_output_from_log-time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \+time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history =get_output_from_log(filename=settings["log_file_name"], time_budget=120)```diff --git a/website/docs/Use-Cases/Tune-User-Defined-Function.md b/website/docs/Use-Cases/Tune-User-Defined-Function.mdindex 9aca4cf62..39f04eb86 100644--- a/website/docs/Use-Cases/Tune-User-Defined-Function.md+++ b/website/docs/Use-Cases/Tune-User-Defined-Function.md@@ -339,7 +339,8 @@ In the following code example, we consider the sample size as the resource dimen```pythonfrom flaml import tunefrom functools import partial-from flaml.data import load_openml_task+from flaml.automl.data import load_openml_task+def obj_from_resource_attr(resource_attr, X_train, X_test, y_train, y_test, config):from lightgbm import LGBMClassifier@@ -362,13 +363,14 @@ def obj_from_resource_attr(resource_attr, X_train, X_test, y_train, y_test, conftest_loss = 1.0 - accuracy_score(y_test, y_test_predict)return {resource_attr: resource, "loss": test_loss}+X_train, X_test, y_train, y_test = load_openml_task(task_id=7592, data_dir="test/")max_resource = len(y_train)resource_attr = "sample_size"min_resource = 1000analysis = tune.run(partial(obj_from_resource_attr, resource_attr, X_train, X_test, y_train, y_test),- config = {+ config={"n_estimators": tune.lograndint(lower=4, upper=32768),"max_leaves": tune.lograndint(lower=4, upper=32768),"learning_rate": tune.loguniform(lower=1 / 1024, upper=1.0),diff --git a/website/docs/Use-Cases/Zero-Shot-AutoML.md b/website/docs/Use-Cases/Zero-Shot-AutoML.mdindex c8871a3fd..bbda22566 100644--- a/website/docs/Use-Cases/Zero-Shot-AutoML.md+++ b/website/docs/Use-Cases/Zero-Shot-AutoML.md@@ -1,6 +1,6 @@# Zero Shot AutoML-`flaml.default` is a package for zero-shot AutoML, or "no-tuning" AutoML. It uses [`flaml.AutoML`](../reference/automl#automl-objects) and [`flaml.default.portfolio`](../reference/default/portfolio) to mine good hyperparameter configurations across different datasets offline, and recommend data-dependent default configurations at runtime without expensive tuning.+`flaml.default` is a package for zero-shot AutoML, or "no-tuning" AutoML. It uses [`flaml.AutoML`](../reference/automl/automl#automl-objects) and [`flaml.default.portfolio`](../reference/default/portfolio) to mine good hyperparameter configurations across different datasets offline, and recommend data-dependent default configurations at runtime without expensive tuning.Zero-shot AutoML has several benefits:* The computation cost is just training one model. No tuning is involved.@@ -48,6 +48,7 @@ Yes. You can use `suggest_hyperparams()` to find the suggested configuration. Fo```pythonfrom flaml.default import LGBMRegressor+estimator = LGBMRegressor()hyperparams, estimator_name, X_transformed, y_transformed = estimator.suggest_hyperparams(X_train, y_train)print(hyperparams)@@ -57,6 +58,7 @@ If you would like more control over the training, use an equivalent, open-box wa```pythonfrom flaml.default import preprocess_and_suggest_hyperparams+X, y = load_iris(return_X_y=True, as_frame=True)X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)hyperparams, estimator_class, X_transformed, y_transformed, feature_transformer, label_transformer = preprocess_and_suggest_hyperparams(@@ -239,6 +241,7 @@ Optionally, you can "flamlize" a learner using [`flaml.default.flamlize_estimato```pythonimport sklearn.ensemble as ensemblefrom flaml.default import flamlize_estimator+ExtraTreesClassifier = flamlize_estimator(ensemble.ExtraTreesClassifier, "extra_tree", "classification")diff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex 227ea0559..fc47c8d44 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -28,7 +28,7 @@ module.exports = {},{type: 'doc',- docId: 'reference/automl',+ docId: 'reference/automl/automl',position: 'left',label: 'SDK',},