A programming framework for agentic AI
Factor out time series-related functionality into a time series Task object (#989)
EgorKraevTransferwise5245efbd2c21
diff --git a/flaml/automl/automl.py b/flaml/automl/automl.pyindex cdf17a37c..c53f70ec8 100644--- a/flaml/automl/automl.py+++ b/flaml/automl/automl.py@@ -13,10 +13,9 @@ import loggingimport jsonfrom flaml.automl.state import SearchState, AutoMLState-from flaml.automl.ml import (- train_estimator,- get_estimator_class,-)+from flaml.automl.ml import train_estimator++from flaml.automl.time_series import TimeSeriesDatasetfrom flaml.config import (MIN_SAMPLE_TRAIN,MEM_THRES,@@ -29,7 +28,7 @@ from flaml.config import ()# TODO check to see when we can remove these-from flaml.automl.task.task import CLASSIFICATION, TS_FORECAST, Task+from flaml.automl.task.task import CLASSIFICATION, Taskfrom flaml.automl.task.factory import task_factoryfrom flaml import tunefrom flaml.automl.logger import logger, logger_formatter@@ -899,7 +898,9 @@ class AutoML(BaseEstimator):], "eval_method must be 'auto' or 'cv' for custom data splitter."assert self._state.X_val is None, "custom splitter and custom validation data can't be used together."return "cv"- if self._state.X_val is not None:+ if self._state.X_val is not None and (+ not isinstance(self._state.X_val, TimeSeriesDataset) or len(self._state.X_val.test_data) > 0+ ):assert eval_method in ["auto","holdout",@@ -1144,7 +1145,7 @@ class AutoML(BaseEstimator):self._df,self._sample_weight_full,)- self.data_size_full = len(self._state.y_train_all)+ self.data_size_full = self._state.data_size_fulldef fit(self,@@ -1196,6 +1197,7 @@ class AutoML(BaseEstimator):free_mem_ratio=0,metric_constraints=None,custom_hp=None,+ time_col=None,cv_score_agg_func=None,skip_transform=None,mlflow_logging=None,@@ -1428,6 +1430,8 @@ class AutoML(BaseEstimator):}}```+ time_col: for a time series task, name of the column containing the timestamps. If not+ provided, defaults to the first column of X_train/X_valcv_score_agg_func: customized cross-validation scores aggregate function. Default to average metrics across folds. If specificed, this function needs tohave the following input arguments:@@ -1522,6 +1526,7 @@ class AutoML(BaseEstimator):if isinstance(task, str):task = task_factory(task, X_train, y_train)self._state.task = task+ self._state.task.time_col = time_colself._estimator_type = "classifier" if task.is_classification() else "regressor"time_budget = time_budget or self._settings.get("time_budget")n_jobs = n_jobs or self._settings.get("n_jobs")@@ -1824,7 +1829,7 @@ class AutoML(BaseEstimator):if estimator_name not in self._state.learner_classes:self.add_learner(estimator_name,- get_estimator_class(self._state.task, estimator_name),+ self._state.task.estimator_class_from_str(estimator_name),)# set up learner search spaceif isinstance(starting_points, str) and starting_points.startswith("data"):@@ -1879,7 +1884,8 @@ class AutoML(BaseEstimator):self._search_states[estimator_name] = SearchState(learner_class=estimator_class,- data_size=self._state.data_size,+ # data_size=self._state.data_size,+ data=self._state.X_train,task=self._state.task,starting_point=starting_points.get(estimator_name),period=self._state.fit_kwargs.get(@@ -2589,7 +2595,7 @@ class AutoML(BaseEstimator):if self._max_iter > 1:self._state.time_budget = -1if (- self._state.task in TS_FORECAST+ self._state.task.is_ts_forecast()or self._trained_estimator is Noneor self._trained_estimator.model is Noneor (diff --git a/flaml/automl/ml.py b/flaml/automl/ml.pyindex 7040088f3..c14ba5cdd 100644--- a/flaml/automl/ml.py+++ b/flaml/automl/ml.py@@ -3,36 +3,16 @@# * Licensed under the MIT License. See LICENSE file in the# * project root for license information.import time-import numpy as npfrom typing import Union, Callable, TypeVar, Optional, Tuple-from flaml.automl.model import (- XGBoostSklearnEstimator,- XGBoost_TS,- XGBoostLimitDepthEstimator,- XGBoostLimitDepth_TS,- RandomForestEstimator,- RF_TS,- LGBMEstimator,- LGBM_TS,- LRL1Classifier,- LRL2Classifier,- CatBoostEstimator,- ExtraTreesEstimator,- ExtraTrees_TS,- KNeighborsEstimator,- Prophet,- ARIMA,- SARIMAX,- HoltWinters,- TransformersEstimator,- TemporalFusionTransformerEstimator,- TransformersEstimatorModelSelection,- SparkLGBMEstimator,-)+import logging++import numpy as np++from flaml.automl.data import group_counts-from flaml.automl.task.task import TS_FORECAST, Task-from flaml.automl.model import BaseEstimator-from flaml.automl.spark import psDataFrame, psSeries, ERROR as SPARK_ERROR, Series+from flaml.automl.task.task import Task+from flaml.automl.model import BaseEstimator, TransformersEstimator+from flaml.automl.spark import psDataFrame, psSeries, ERROR as SPARK_ERROR, Series, DataFrametry:from sklearn.metrics import (@@ -53,6 +33,11 @@ except ImportError:if SPARK_ERROR is None:from flaml.automl.spark.metrics import spark_metric_loss_score+from flaml.automl.time_series import TimeSeriesDataset++logger = logging.getLogger(__name__)++EstimatorSubclass = TypeVar("EstimatorSubclass", bound=BaseEstimator)sklearn_metric_name_set = {@@ -106,55 +91,6 @@ huggingface_metric_to_mode = {huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}-def get_estimator_class(task: str, estimator_name: str) -> EstimatorSubclass:- """Given a task and an estimator name, return the relevant flaml-wrapped estimator class-- NOTE: See why the return type is declarad by using TypeVar here on the mypy doc- https://mypy.readthedocs.io/en/stable/kinds_of_types.html#the-type-of-class-objects- """- # when adding a new learner, need to add an elif branch- if "xgboost" == estimator_name:- estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator- elif "xgb_limitdepth" == estimator_name:- estimator_class = XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator- elif "rf" == estimator_name:- estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator- elif "lgbm" == estimator_name:- estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator- elif "lgbm_spark" == estimator_name:- estimator_class = SparkLGBMEstimator- elif "lrl1" == estimator_name:- estimator_class = LRL1Classifier- elif "lrl2" == estimator_name:- estimator_class = LRL2Classifier- elif "catboost" == estimator_name:- estimator_class = CatBoostEstimator- elif "extra_tree" == estimator_name:- estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator- elif "kneighbor" == estimator_name:- estimator_class = KNeighborsEstimator- elif "prophet" in estimator_name:- estimator_class = Prophet- elif estimator_name == "arima":- estimator_class = ARIMA- elif estimator_name == "sarimax":- estimator_class = SARIMAX- elif estimator_name == "holt-winters":- estimator_class = HoltWinters- elif estimator_name == "transformer":- estimator_class = TransformersEstimator- elif estimator_name == "tft":- estimator_class = TemporalFusionTransformerEstimator- elif estimator_name == "transformer_ms":- estimator_class = TransformersEstimatorModelSelection- else:- raise ValueError(- estimator_name + " is not a built-in learner. "- "Please use AutoML.add_learner() to add a customized learner."- )- return estimator_class--def metric_loss_score(metric_name: str,y_processed_predict,@@ -267,6 +203,7 @@ def sklearn_metric_loss_score(Returns:score: A float number of the loss, the lower the better."""+metric_name = metric_name.lower()if "r2" == metric_name:@@ -356,132 +293,20 @@ def get_y_pred(estimator, X, eval_metric, task: Task):y_pred = estimator.predict_proba(X)else:y_pred = estimator.predict(X)- return y_pred+ if isinstance(y_pred, Series) or isinstance(y_pred, DataFrame):+ y_pred = y_pred.values-def _eval_estimator(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric: Union[str, Callable],- task,- labels=None,- log_training_metric=False,- fit_kwargs: Optional[dict] = None,-):- if fit_kwargs is None:- fit_kwargs = {}- if isinstance(eval_metric, str):- pred_start = time.time()- val_pred_y = get_y_pred(estimator, X_val, eval_metric, task)- pred_time = (time.time() - pred_start) / X_val.shape[0]+ return y_pred- val_loss = metric_loss_score(- eval_metric,- y_processed_predict=val_pred_y,- y_processed_true=y_val,- labels=labels,- sample_weight=weight_val,- groups=groups_val,- )- metric_for_logging = {"pred_time": pred_time}- if log_training_metric:- train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)- metric_for_logging["train_loss"] = metric_loss_score(- eval_metric,- train_pred_y,- y_train,- labels,- fit_kwargs.get("sample_weight"),- fit_kwargs.get("groups"),- )- else: # customized metric function- val_loss, metric_for_logging = eval_metric(- X_val,- y_val,- estimator,- labels,- X_train,- y_train,- weight_val,- fit_kwargs.get("sample_weight"),- config,- groups_val,- fit_kwargs.get("groups"),- )- pred_time = metric_for_logging.get("pred_time", 0)- val_pred_y = None- # eval_metric may return val_pred_y but not necessarily. Setting None for now.- return val_loss, metric_for_logging, pred_time, val_pred_y+def to_numpy(x):+ if isinstance(x, Series or isinstance(x, DataFrame)):+ x = x.values+ else:+ x = np.ndarray(x)-def get_val_loss(- config,- estimator: EstimatorSubclass,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric: Union[str, Callable],- obj,- labels=None,- budget=None,- log_training_metric=False,- fit_kwargs: Optional[dict] = None,- free_mem_ratio=0,-):- if fit_kwargs is None:- fit_kwargs = {}- start = time.time()- # if groups_val is not None:- # fit_kwargs['groups_val'] = groups_val- # fit_kwargs['X_val'] = X_val- # fit_kwargs['y_val'] = y_val- estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)- val_loss, metric_for_logging, pred_time, _ = _eval_estimator(- config,- estimator,- X_train,- y_train,- X_val,- y_val,- weight_val,- groups_val,- eval_metric,- obj,- labels,- log_training_metric,- fit_kwargs,- )- if hasattr(estimator, "intermediate_results"):- metric_for_logging["intermediate_results"] = estimator.intermediate_results- train_time = time.time() - start- return val_loss, metric_for_logging, train_time, pred_time---def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):- metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)- metrics_to_log = None- for single_fold in log_metrics_folds:- if metrics_to_log is None:- metrics_to_log = single_fold- elif isinstance(metrics_to_log, dict):- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}- else:- metrics_to_log += single_fold- if metrics_to_log:- n = len(val_loss_folds)- metrics_to_log = (- {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n- )- return metric_to_minimize, metrics_to_log+ return x.reshape((-1, 1))def compute_estimator(@@ -494,7 +319,7 @@ def compute_estimator(budget,kf,config_dic: dict,- task: str,+ task: Union[str, Task],estimator_name: str,eval_method: str,eval_metric: Union[str, Callable],@@ -509,7 +334,7 @@ def compute_estimator(if fit_kwargs is None:fit_kwargs = {}- estimator_class = estimator_class or get_estimator_class(task, estimator_name)+ estimator_class = estimator_class or task.estimator_class_from_str(estimator_name)estimator = estimator_class(**config_dic,task=task,@@ -576,7 +401,7 @@ def train_estimator(free_mem_ratio=0,) -> Tuple[EstimatorSubclass, float]:start_time = time.time()- estimator_class = estimator_class or get_estimator_class(task, estimator_name)+ estimator_class = estimator_class or task.estimator_class_from_str(estimator_name)estimator = estimator_class(**config_dic,task=task,@@ -589,7 +414,7 @@ def train_estimator(fit_kwargs["metric"] = eval_metricif X_train is not None:- train_time = estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)+ train_time = estimator.fit(X_train, y_train, budget=budget, free_mem_ratio=free_mem_ratio, **fit_kwargs)else:estimator = estimator.estimator_class(**estimator.params)train_time = time.time() - start_time@@ -642,3 +467,140 @@ def multi_class_curves(for i in range(len(classes)):curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])return curve_x, curve_y+++def get_val_loss(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ task,+ labels=None,+ budget=None,+ log_training_metric=False,+ fit_kwargs={},+ free_mem_ratio=0,+):+ start = time.time()+ # if groups_val is not None:+ # fit_kwargs['groups_val'] = groups_val+ # fit_kwargs['X_val'] = X_val+ # fit_kwargs['y_val'] = y_val+ estimator.fit(X_train, y_train, budget=budget, free_mem_ratio=free_mem_ratio, **fit_kwargs)+ val_loss, metric_for_logging, pred_time, _ = _eval_estimator(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ task,+ labels,+ log_training_metric,+ fit_kwargs,+ )+ if hasattr(estimator, "intermediate_results"):+ metric_for_logging["intermediate_results"] = estimator.intermediate_results+ train_time = time.time() - start+ return val_loss, metric_for_logging, train_time, pred_time+++def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):+ metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)+ metrics_to_log = None+ for single_fold in log_metrics_folds:+ if metrics_to_log is None:+ metrics_to_log = single_fold+ elif isinstance(metrics_to_log, dict):+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}+ else:+ metrics_to_log += single_fold+ if metrics_to_log:+ n = len(val_loss_folds)+ metrics_to_log = (+ {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n+ )+ return metric_to_minimize, metrics_to_log+++def _eval_estimator(+ config,+ estimator,+ X_train,+ y_train,+ X_val,+ y_val,+ weight_val,+ groups_val,+ eval_metric,+ task,+ labels=None,+ log_training_metric=False,+ fit_kwargs={},+):+ if isinstance(eval_metric, str):+ pred_start = time.time()+ val_pred_y = get_y_pred(estimator, X_val, eval_metric, task)++ # TODO: why are integer labels being cast to str in the first place?++ if isinstance(val_pred_y, Series) or isinstance(val_pred_y, DataFrame) or isinstance(val_pred_y, np.ndarray):+ test = val_pred_y if isinstance(val_pred_y, np.ndarray) else val_pred_y.values+ if not np.issubdtype(test.dtype, np.number):+ # some NLP models return a list+ val_pred_y = val_pred_y.astype(str)++ if isinstance(X_val, TimeSeriesDataset):+ num_val_rows = len(X_val.test_data)+ y_val = X_val.test_data[X_val.target_names].values.astype(val_pred_y.dtype)+ y_train = X_val.train_data[X_val.target_names].values.astype(val_pred_y.dtype)+ else:+ num_val_rows = X_val.shape[0]++ pred_time = (time.time() - pred_start) / num_val_rows++ val_loss = metric_loss_score(+ eval_metric,+ y_processed_predict=val_pred_y,+ y_processed_true=y_val,+ labels=labels,+ sample_weight=weight_val,+ groups=groups_val,+ )+ metric_for_logging = {"pred_time": pred_time}+ if log_training_metric:+ train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)+ metric_for_logging["train_loss"] = metric_loss_score(+ eval_metric,+ train_pred_y,+ y_train,+ labels,+ fit_kwargs.get("sample_weight"),+ fit_kwargs.get("groups"),+ )+ else: # customized metric function+ val_loss, metric_for_logging = eval_metric(+ X_val,+ y_val,+ estimator,+ labels,+ X_train,+ y_train,+ weight_val,+ fit_kwargs.get("sample_weight"),+ config,+ groups_val,+ fit_kwargs.get("groups"),+ )+ pred_time = metric_for_logging.get("pred_time", 0)+ val_pred_y = None+ # eval_metric may return val_pred_y but not necessarily. Setting None for now.+ return val_loss, metric_for_logging, pred_time, val_pred_ydiff --git a/flaml/automl/model.py b/flaml/automl/model.pyindex b1be92f15..fea6e8a6d 100644--- a/flaml/automl/model.py+++ b/flaml/automl/model.py@@ -16,19 +16,16 @@ import mathfrom flaml import tunefrom flaml.automl.data import (group_counts,- add_time_idx_col,- TS_TIMESTAMP_COL,- TS_VALUE_COL,)from flaml.automl.task.task import (- CLASSIFICATION,- TS_FORECASTREGRESSION,+ Task,SEQCLASSIFICATION,SEQREGRESSION,TOKENCLASSIFICATION,SUMMARIZATION,NLG_TASKS,)+from flaml.automl.task.factory import task_factorytry:from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier@@ -119,13 +116,13 @@ class BaseEstimator:config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.n_jobs is the number of parallel threads."""- self._task = task+ self._task = task if isinstance(task, Task) else task_factory(task, None, None)self.params = self.config2params(config)self.estimator_class = self._model = Noneif "_estimator_type" in config:self._estimator_type = self.params.pop("_estimator_type")else:- self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor"+ self._estimator_type = "classifier" if self._task.is_classification() else "regressor"def get_params(self, deep=False):params = self.params.copy()@@ -247,7 +244,7 @@ class BaseEstimator:train_time = self._fit(X_train, y_train, **kwargs)except (MemoryError, TimeoutError) as e:logger.warning(f"{e.__class__} {e}")- if self._task in CLASSIFICATION:+ if self._task.is_classification():model = DummyClassifier()else:model = DummyRegressor()@@ -289,7 +286,7 @@ class BaseEstimator:Each element at (i,j) is the probability for instance i to be inclass j."""- assert self._task in CLASSIFICATION, "predict_proba() only for classification."+ assert self._task.is_classification(), "predict_proba() only for classification."X = self._preprocess(X)return self._model.predict_proba(X, **kwargs)@@ -490,7 +487,7 @@ class SparkEstimator(BaseEstimator):Each element at (i,j) is the probability for instance i to be inclass j."""- assert self._task in CLASSIFICATION, "predict_proba() only for classification."+ assert self._task.is_classification(), "predict_proba() only for classification."if self._model is not None:X = self._preprocess(X, index_col=index_col)predictions = to_pandas_on_spark(self._model.transform(X), index_col=index_col)@@ -1111,7 +1108,7 @@ class TransformersEstimator(BaseEstimator):for key, val in pred_kwargs.items():setattr(self._training_args, key, val)- assert self._task in CLASSIFICATION, "predict_proba() only for classification tasks."+ assert self._task.is_classification(), "predict_proba() only for classification tasks."X_test, _ = self._tokenize_text(X, **self._kwargs)test_dataset = Dataset.from_pandas(X_test)@@ -1297,18 +1294,21 @@ class LGBMEstimator(BaseEstimator):super().__init__(task, **config)if "verbose" not in self.params:self.params["verbose"] = -1- if "regression" == task:- from lightgbm import LGBMRegressor- self.estimator_class = LGBMRegressor- elif "rank" == task:+ if self._task.is_classification():+ from lightgbm import LGBMClassifier++ self.estimator_class = LGBMClassifier++ elif task == "rank":from lightgbm import LGBMRankerself.estimator_class = LGBMRankerelse:- from lightgbm import LGBMClassifier+ from lightgbm import LGBMRegressor++ self.estimator_class = LGBMRegressor- self.estimator_class = LGBMClassifierself._time_per_iter = Noneself._train_size = 0self._mem_per_iter = -1@@ -1626,11 +1626,13 @@ class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):self.params["verbosity"] = 0import xgboost as xgb- self.estimator_class = xgb.XGBRegressorif "rank" == task:self.estimator_class = xgb.XGBRanker- elif task in CLASSIFICATION:+ elif self._task.is_classification():self.estimator_class = xgb.XGBClassifier+ else:+ self.estimator_class = xgb.XGBRegressor+self._xgb_version = xgb.__version__def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):@@ -1678,7 +1680,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):def search_space(cls, data_size, task, **params):RandomForestEstimator.nrows = int(data_size[0])upper = min(2048, RandomForestEstimator.nrows)- init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1+ init = 1 / np.sqrt(data_size[1]) if task.is_classification() else 1lower = min(0.1, init)space = {"n_estimators": {@@ -1699,7 +1701,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):"low_cost_init_value": 4,},}- if task in CLASSIFICATION:+ if task.is_classification():space["criterion"] = {"domain": tune.choice(["gini", "entropy"]),# "init_value": "gini",@@ -1714,7 +1716,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):params = super().config2params(config)if "max_leaves" in params:params["max_leaf_nodes"] = params.get("max_leaf_nodes", params.pop("max_leaves"))- if self._task not in CLASSIFICATION and "criterion" in config:+ if not self._task.is_classification() and "criterion" in config:params.pop("criterion")if "random_state" not in params:params["random_state"] = 12032022@@ -1722,14 +1724,16 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):def __init__(self,- task="binary",+ task: Task,**params,):super().__init__(task, **params)self.params["verbose"] = 0- self.estimator_class = RandomForestRegressor- if task in CLASSIFICATION:++ if self._task.is_classification():self.estimator_class = RandomForestClassifier+ else:+ self.estimator_class = RandomForestRegressorclass ExtraTreesEstimator(RandomForestEstimator):@@ -1776,7 +1780,7 @@ class LRL1Classifier(SKLearnEstimator):def __init__(self, task="binary", **config):super().__init__(task, **config)- assert task in CLASSIFICATION, "LogisticRegression for classification task only"+ assert self._task.is_classification(), "LogisticRegression for classification task only"self.estimator_class = LogisticRegression@@ -1802,7 +1806,7 @@ class LRL2Classifier(SKLearnEstimator):def __init__(self, task="binary", **config):super().__init__(task, **config)- assert task in CLASSIFICATION, "LogisticRegression for classification task only"+ assert self._task.is_classification(), "LogisticRegression for classification task only"self.estimator_class = LogisticRegression@@ -1877,13 +1881,14 @@ class CatBoostEstimator(BaseEstimator):"random_seed": config.get("random_seed", 10242048),})- from catboost import CatBoostRegressor-- self.estimator_class = CatBoostRegressor- if task in CLASSIFICATION:+ if self._task.is_classification():from catboost import CatBoostClassifierself.estimator_class = CatBoostClassifier+ else:+ from catboost import CatBoostRegressor++ self.estimator_class = CatBoostRegressordef fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):start_time = time.time()@@ -1976,7 +1981,7 @@ class KNeighborsEstimator(BaseEstimator):def __init__(self, task="binary", **config):super().__init__(task, **config)- if task in CLASSIFICATION:+ if self._task.is_classification():from sklearn.neighbors import KNeighborsClassifierself.estimator_class = KNeighborsClassifier@@ -2003,660 +2008,6 @@ class KNeighborsEstimator(BaseEstimator):return X-class Prophet(SKLearnEstimator):- """The class for tuning Prophet."""-- @classmethod- def search_space(cls, **params):- space = {- "changepoint_prior_scale": {- "domain": tune.loguniform(lower=0.001, upper=0.05),- "init_value": 0.05,- "low_cost_init_value": 0.001,- },- "seasonality_prior_scale": {- "domain": tune.loguniform(lower=0.01, upper=10),- "init_value": 10,- },- "holidays_prior_scale": {- "domain": tune.loguniform(lower=0.01, upper=10),- "init_value": 10,- },- "seasonality_mode": {- "domain": tune.choice(["additive", "multiplicative"]),- "init_value": "multiplicative",- },- }- return space-- def __init__(self, task="ts_forecast", n_jobs=1, **params):- super().__init__(task, **params)-- def _join(self, X_train, y_train):- assert TS_TIMESTAMP_COL in X_train, (- "Dataframe for training ts_forecast model must have column"- f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'- )- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])- train_df = X_train.join(y_train)- return train_df-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- from prophet import Prophet-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- cols = list(train_df)- cols.remove(TS_TIMESTAMP_COL)- cols.remove(TS_VALUE_COL)- logging.getLogger("prophet").setLevel(logging.WARNING)- model = Prophet(**self.params)- for regressor in cols:- model.add_regressor(regressor)- with suppress_stdout_stderr():- model.fit(train_df)- train_time = time.time() - current_time- self._model = model- return train_time-- def predict(self, X, **kwargs):- if isinstance(X, int):- raise ValueError(- "predict() with steps is only supported for arima/sarimax."- " For Prophet, pass a dataframe with the first column containing"- " the timestamp values."- )- if self._model is not None:- X = self._preprocess(X)- forecast = self._model.predict(X, **kwargs)- return forecast["yhat"]- else:- logger.warning("Estimator is not fit yet. Please run fit() before predict().")- return np.ones(X.shape[0])-- def score(self, X_val: DataFrame, y_val: Series, **kwargs):- from sklearn.metrics import r2_score- from .ml import metric_loss_score-- y_pred = self.predict(X_val, **kwargs)- self._metric = kwargs.get("metric", None)- if self._metric:- return metric_loss_score(self._metric, y_pred, y_val)- else:- return r2_score(y_pred, y_val)---class ARIMA(Prophet):- """The class for tuning ARIMA."""-- @classmethod- def search_space(cls, **params):- space = {- "p": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "d": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- }- return space-- def _join(self, X_train, y_train):- train_df = super()._join(X_train, y_train)- train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])- train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)- return train_df-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings-- warnings.filterwarnings("ignore")- from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- regressors = list(train_df)- regressors.remove(TS_VALUE_COL)- if regressors:- model = ARIMA_estimator(- train_df[[TS_VALUE_COL]],- exog=train_df[regressors],- order=(self.params["p"], self.params["d"], self.params["q"]),- enforce_stationarity=False,- enforce_invertibility=False,- )- else:- model = ARIMA_estimator(- train_df,- order=(self.params["p"], self.params["d"], self.params["q"]),- enforce_stationarity=False,- enforce_invertibility=False,- )- with suppress_stdout_stderr():- model = model.fit()- train_time = time.time() - current_time- self._model = model- return train_time-- def predict(self, X, **kwargs):- if self._model is not None:- if isinstance(X, int):- forecast = self._model.forecast(steps=X)- elif isinstance(X, DataFrame):- start = X[TS_TIMESTAMP_COL].iloc[0]- end = X[TS_TIMESTAMP_COL].iloc[-1]- if len(X.columns) > 1:- X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))- regressors = list(X)- forecast = self._model.predict(start=start, end=end, exog=X[regressors], **kwargs)- else:- forecast = self._model.predict(start=start, end=end, **kwargs)- else:- raise ValueError(- "X needs to be either a pandas Dataframe with dates as the first column"- " or an int number of periods for predict()."- )- return forecast- else:- return np.ones(X if isinstance(X, int) else X.shape[0])---class SARIMAX(ARIMA):- """The class for tuning SARIMA."""-- @classmethod- def search_space(cls, **params):- space = {- "p": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "d": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 2,- "low_cost_init_value": 0,- },- "q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "P": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "D": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "Q": {- "domain": tune.qrandint(lower=0, upper=10, q=1),- "init_value": 1,- "low_cost_init_value": 0,- },- "s": {- "domain": tune.choice([1, 4, 6, 12]),- "init_value": 12,- },- }- return space-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings-- warnings.filterwarnings("ignore")- from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- regressors = list(train_df)- regressors.remove(TS_VALUE_COL)- if regressors:- model = SARIMAX_estimator(- train_df[[TS_VALUE_COL]],- exog=train_df[regressors],- order=(self.params["p"], self.params["d"], self.params["q"]),- seasonal_order=(- self.params["P"],- self.params["D"],- self.params["Q"],- self.params["s"],- ),- enforce_stationarity=False,- enforce_invertibility=False,- )- else:- model = SARIMAX_estimator(- train_df,- order=(self.params["p"], self.params["d"], self.params["q"]),- seasonal_order=(- self.params["P"],- self.params["D"],- self.params["Q"],- self.params["s"],- ),- enforce_stationarity=False,- enforce_invertibility=False,- )- with suppress_stdout_stderr():- model = model.fit()- train_time = time.time() - current_time- self._model = model- return train_time---class HoltWinters(ARIMA):- """- The class for tuning Holt Winters model, aka 'Triple Exponential Smoothing'.- """-- @classmethod- def search_space(cls, **params):- space = {- "damped_trend": {"domain": tune.choice([True, False]), "init_value": False},- "trend": {"domain": tune.choice(["add", "mul", None]), "init_value": "add"},- "seasonal": {- "domain": tune.choice(["add", "mul", None]),- "init_value": "add",- },- "use_boxcox": {"domain": tune.choice([False, True]), "init_value": False},- "seasonal_periods": { # statsmodels casts this to None if "seasonal" is None- "domain": tune.choice([7, 12, 4, 52, 6]), # weekly, yearly, quarterly, weekly w yearly data- "init_value": 7,- },- }- return space-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings-- warnings.filterwarnings("ignore")- from statsmodels.tsa.holtwinters import (- ExponentialSmoothing as HWExponentialSmoothing,- )-- current_time = time.time()- train_df = self._join(X_train, y_train)- train_df = self._preprocess(train_df)- regressors = list(train_df)- regressors.remove(TS_VALUE_COL)- if regressors:- logger.warning("Regressors are ignored for Holt-Winters ETS models.")-- # Override incompatible parameters- if (- X_train.shape[0] < 2 * self.params["seasonal_periods"]- ): # this would prevent heuristic initialization to work properly- self.params["seasonal"] = None- if (- self.params["seasonal"] == "mul" and (train_df.y == 0).sum() > 0- ): # cannot have multiplicative seasonality in this case- self.params["seasonal"] = "add"- if self.params["trend"] == "mul" and (train_df.y == 0).sum() > 0:- self.params["trend"] = "add"-- if not self.params["seasonal"] or self.params["trend"] not in ["mul", "add"]:- self.params["damped_trend"] = False-- model = HWExponentialSmoothing(- train_df[[TS_VALUE_COL]],- damped_trend=self.params["damped_trend"],- seasonal=self.params["seasonal"],- trend=self.params["trend"],- )- with suppress_stdout_stderr():- model = model.fit()- train_time = time.time() - current_time- self._model = model- return train_time-- def predict(self, X, **kwargs):- if self._model is not None:- if isinstance(X, int):- forecast = self._model.forecast(steps=X)- elif isinstance(X, DataFrame):- start = X[TS_TIMESTAMP_COL].iloc[0]- end = X[TS_TIMESTAMP_COL].iloc[-1]- forecast = self._model.predict(start=start, end=end, **kwargs)- else:- raise ValueError(- "X needs to be either a pandas Dataframe with dates as the first column"- " or an int number of periods for predict()."- )- return forecast- else:- return np.ones(X if isinstance(X, int) else X.shape[0])---class TS_SKLearn(SKLearnEstimator):- """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""-- base_class = SKLearnEstimator-- @classmethod- def search_space(cls, data_size, pred_horizon, **params):- space = cls.base_class.search_space(data_size, **params)- space.update(- {- "optimize_for_horizon": {- "domain": tune.choice([True, False]),- "init_value": False,- "low_cost_init_value": False,- },- "lags": {- "domain": tune.randint(lower=1, upper=max(2, int(np.sqrt(data_size[0])))),- "init_value": 3,- },- }- )- return space-- def __init__(self, task="ts_forecast", **params):- super().__init__(task, **params)- self.hcrystaball_model = None- self.ts_task = "regression" if task in TS_FORECASTREGRESSION else "classification"-- def transform_X(self, X):- cols = list(X)- if len(cols) == 1:- ds_col = cols[0]- X = DataFrame(index=X[ds_col])- elif len(cols) > 1:- ds_col = cols[0]- exog_cols = cols[1:]- X = X[exog_cols].set_index(X[ds_col])- return X-- def _fit(self, X_train, y_train, budget=None, **kwargs):- from hcrystalball.wrappers import get_sklearn_wrapper-- X_train = self.transform_X(X_train)- X_train = self._preprocess(X_train)- params = self.params.copy()- lags = params.pop("lags")- optimize_for_horizon = params.pop("optimize_for_horizon")- estimator = self.base_class(task=self.ts_task, **params)- self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)- self.hcrystaball_model.lags = int(lags)- self.hcrystaball_model.fit(X_train, y_train)- if optimize_for_horizon:- # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon- model_list = []- for i in range(1, kwargs["period"] + 1):- (- X_fit,- y_fit,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X_train, y_train, i)- self.hcrystaball_model.model.set_params(**estimator.params)- model = self.hcrystaball_model.model.fit(X_fit, y_fit)- model_list.append(model)- self._model = model_list- else:- (- X_fit,- y_fit,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X_train, y_train, kwargs["period"])- self.hcrystaball_model.model.set_params(**estimator.params)- model = self.hcrystaball_model.model.fit(X_fit, y_fit)- self._model = model-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- current_time = time.time()- self._fit(X_train, y_train, budget=budget, **kwargs)- train_time = time.time() - current_time- return train_time-- def predict(self, X, **kwargs):- if self._model is not None:- X = self.transform_X(X)- X = self._preprocess(X)- if isinstance(self._model, list):- assert len(self._model) == len(- X- ), "Model is optimized for horizon, length of X must be equal to `period`."- preds = []- for i in range(1, len(self._model) + 1):- (- X_pred,- _,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X.iloc[:i, :])- preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])- forecast = Series(preds)- else:- (- X_pred,- _,- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)- forecast = self._model.predict(X_pred, **kwargs)- return forecast- else:- logger.warning("Estimator is not fit yet. Please run fit() before predict().")- return np.ones(X.shape[0])---class LGBM_TS(TS_SKLearn):- """The class for tuning LGBM Regressor for time-series forecasting"""-- base_class = LGBMEstimator---class XGBoost_TS(TS_SKLearn):- """The class for tuning XGBoost Regressor for time-series forecasting"""-- base_class = XGBoostSklearnEstimator---# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball-# class CatBoost_TS_Regressor(TS_Regressor):-# base_class = CatBoostEstimator---class RF_TS(TS_SKLearn):- """The class for tuning Random Forest Regressor for time-series forecasting"""-- base_class = RandomForestEstimator---class ExtraTrees_TS(TS_SKLearn):- """The class for tuning Extra Trees Regressor for time-series forecasting"""-- base_class = ExtraTreesEstimator---class XGBoostLimitDepth_TS(TS_SKLearn):- """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""-- base_class = XGBoostLimitDepthEstimator---class TemporalFusionTransformerEstimator(SKLearnEstimator):- """The class for tuning Temporal Fusion Transformer"""-- @classmethod- def search_space(cls, data_size, pred_horizon, **params):- space = {- "gradient_clip_val": {- "domain": tune.loguniform(lower=0.01, upper=100.0),- "init_value": 0.01,- },- "hidden_size": {- "domain": tune.lograndint(lower=8, upper=512),- "init_value": 16,- },- "hidden_continuous_size": {- "domain": tune.randint(lower=1, upper=65),- "init_value": 8,- },- "attention_head_size": {- "domain": tune.randint(lower=1, upper=5),- "init_value": 4,- },- "dropout": {- "domain": tune.uniform(lower=0.1, upper=0.3),- "init_value": 0.1,- },- "learning_rate": {- "domain": tune.loguniform(lower=0.00001, upper=1.0),- "init_value": 0.001,- },- }- return space-- def transform_ds(self, X_train, y_train, **kwargs):- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])- self.data = X_train.join(y_train)-- max_prediction_length = kwargs["period"]- self.max_encoder_length = kwargs["max_encoder_length"]- training_cutoff = self.data["time_idx"].max() - max_prediction_length-- from pytorch_forecasting import TimeSeriesDataSet- from pytorch_forecasting.data import GroupNormalizer-- self.group_ids = kwargs["group_ids"].copy()- training = TimeSeriesDataSet(- self.data[lambda x: x.time_idx <= training_cutoff],- time_idx="time_idx",- target=TS_VALUE_COL,- group_ids=self.group_ids,- min_encoder_length=kwargs.get(- "min_encoder_length", self.max_encoder_length // 2- ), # keep encoder length long (as it is in the validation set)- max_encoder_length=self.max_encoder_length,- min_prediction_length=1,- max_prediction_length=max_prediction_length,- static_categoricals=kwargs.get("static_categoricals", []),- static_reals=kwargs.get("static_reals", []),- time_varying_known_categoricals=kwargs.get("time_varying_known_categoricals", []),- time_varying_known_reals=kwargs.get("time_varying_known_reals", []),- time_varying_unknown_categoricals=kwargs.get("time_varying_unknown_categoricals", []),- time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),- variable_groups=kwargs.get(- "variable_groups", {}- ), # group of categorical variables can be treated as one variable- lags=kwargs.get("lags", {}),- target_normalizer=GroupNormalizer(- groups=kwargs["group_ids"], transformation="softplus"- ), # use softplus and normalize by group- add_relative_time_idx=True,- add_target_scales=True,- add_encoder_length=True,- )-- # create validation set (predict=True) which means to predict the last max_prediction_length points in time- # for each series- validation = TimeSeriesDataSet.from_dataset(training, self.data, predict=True, stop_randomization=True)-- # create dataloaders for model- batch_size = kwargs.get("batch_size", 64)- train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)- val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size * 10, num_workers=0)-- return training, train_dataloader, val_dataloader-- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):- import warnings- import pytorch_lightning as pl- from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor- import torch- from pytorch_forecasting import TemporalFusionTransformer- from pytorch_forecasting.metrics import QuantileLoss-- warnings.filterwarnings("ignore")- current_time = time.time()- training, train_dataloader, val_dataloader = self.transform_ds(X_train, y_train, **kwargs)- params = self.params.copy()- gradient_clip_val = params.pop("gradient_clip_val")- params.pop("n_jobs")- max_epochs = kwargs.get("max_epochs", 20)- early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")-- def _fit(log):- default_trainer_kwargs = dict(- gpus=kwargs.get("gpu_per_trial", [0]) if torch.cuda.is_available() else None,- max_epochs=max_epochs,- gradient_clip_val=gradient_clip_val,- callbacks=[LearningRateMonitor(), early_stop_callback] if log else [early_stop_callback],- logger=log,- )- trainer = pl.Trainer(- **default_trainer_kwargs,- )- tft = TemporalFusionTransformer.from_dataset(- training,- **params,- lstm_layers=2, # 2 is mostly optimal according to documentation- output_size=7, # 7 quantiles by default- loss=QuantileLoss(),- log_interval=10 if log else 0,- # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches- reduce_on_plateau_patience=4,- )- # fit network- trainer.fit(- tft,- train_dataloaders=train_dataloader,- val_dataloaders=val_dataloader,- )- return trainer-- # try:- # from pytorch_lightning.loggers import TensorBoardLogger-- # logger = TensorBoardLogger(- # kwargs.get("log_dir", "lightning_logs")- # ) # logging results to a tensorboard- # trainer = _fit(log=logger)- # except ValueError:- # issue with pytorch forecasting model log_prediction() function- # pytorch-forecasting issue #1145- trainer = _fit(log=False)- best_model_path = trainer.checkpoint_callback.best_model_path- best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)- train_time = time.time() - current_time- self._model = best_tft- return train_time-- def predict(self, X):- import pandas as pd-- ids = self.group_ids.copy()- ids.append(TS_TIMESTAMP_COL)- encoder_data = self.data[lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length]- # following pytorchforecasting example, make all target values equal to the last data- last_data_cols = self.group_ids.copy()- last_data_cols.append(TS_VALUE_COL)- last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]- decoder_data = X- if "time_idx" not in decoder_data:- decoder_data = add_time_idx_col(decoder_data)- decoder_data["time_idx"] += encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()- # decoder_data[TS_VALUE_COL] = 0- decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)- decoder_data = decoder_data.sort_values(ids)- new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)- new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")- new_raw_predictions = self._model.predict(new_prediction_data)- index = [decoder_data[idx].to_numpy() for idx in ids]- predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)- return predictions--class suppress_stdout_stderr(object):def __init__(self):# Open a pair of null filesdiff --git a/flaml/automl/state.py b/flaml/automl/state.pyindex 1597271a1..1b473b75d 100644--- a/flaml/automl/state.py+++ b/flaml/automl/state.py@@ -1,11 +1,12 @@import inspect+import copyimport timefrom typing import Any, Optionalimport numpy as npfrom flaml import tunefrom flaml.automl.logger import loggerfrom flaml.automl.ml import compute_estimator, train_estimator-from flaml.automl.task.task import TS_FORECAST+from flaml.automl.time_series.ts_data import TimeSeriesDatasetfrom flaml.automl.spark import psDataFrame, psSeries, DataFrame, Series@@ -55,7 +56,7 @@ class SearchState:def __init__(self,learner_class,- data_size,+ data,task,starting_point=None,period=None,@@ -68,14 +69,18 @@ class SearchState:self.init_config = Noneself.low_cost_partial_config = {}self.cat_hp_cost = {}- self.data_size = data_size+self.ls_ever_converged = Falseself.learner_class = learner_classself._budget = budget- if task in TS_FORECAST:- search_space = learner_class.search_space(data_size=data_size, task=task, pred_horizon=period)++ if task.is_ts_forecast():+ data_size = data.train_data.shape+ search_space = learner_class.search_space(data=data, task=task, pred_horizon=period)else:+ data_size = data.shapesearch_space = learner_class.search_space(data_size=data_size, task=task)+ self.data_size = data_sizeif custom_hp is not None:search_space.update(custom_hp)@@ -211,17 +216,22 @@ class SearchState:class AutoMLState:- def _prepare_sample_train_data(self, sample_size: int):+ def prepare_sample_train_data(self, sample_size: int):sampled_weight = groups = Noneif sample_size <= self.data_size[0]:- if isinstance(self.X_train, (DataFrame, psDataFrame)):- sampled_X_train = self.X_train.iloc[:sample_size]+ if isinstance(self.X_train, TimeSeriesDataset):+ sampled_X_train = copy.copy(self.X_train)+ sampled_X_train.train_data = self.X_train.train_data.iloc[-sample_size:]+ sampled_y_train = Noneelse:- sampled_X_train = self.X_train[:sample_size]- if isinstance(self.y_train, (Series, psSeries)):- sampled_y_train = self.y_train.iloc[:sample_size]- else:- sampled_y_train = self.y_train[:sample_size]+ if isinstance(self.X_train, (DataFrame, psDataFrame)):+ sampled_X_train = self.X_train.iloc[:sample_size]+ else:+ sampled_X_train = self.X_train[:sample_size]+ if isinstance(self.y_train, (Series, psSeries)):+ sampled_y_train = self.y_train.iloc[:sample_size]+ else:+ sampled_y_train = self.y_train[:sample_size]weight = self.fit_kwargs.get("sample_weight") # NOTE: _prepare_sample_train_data is before kwargs is updated to fit_kwargs_by_estimator@@ -266,7 +276,7 @@ class AutoMLState:sampled_y_train,sampled_weight,groups,- ) = state._prepare_sample_train_data(sample_size)+ ) = state.task.prepare_sample_train_data(state, sample_size)if sampled_weight is not None:weight = this_estimator_kwargs["sample_weight"]this_estimator_kwargs["sample_weight"] = sampled_weight@@ -354,7 +364,7 @@ class AutoMLState:sampled_y_train,sampled_weight,groups,- ) = self._prepare_sample_train_data(sample_size)+ ) = self.task.prepare_sample_train_data(self, sample_size)if sampled_weight is not None:weight = this_estimator_kwargs["sample_weight"diff --git a/flaml/automl/task/__init__.py b/flaml/automl/task/__init__.pyindex e69de29bb..280e6a2ad 100644--- a/flaml/automl/task/__init__.py+++ b/flaml/automl/task/__init__.py@@ -0,0 +1 @@+from .task import Taskdiff --git a/flaml/automl/task/factory.py b/flaml/automl/task/factory.pyindex 7974b9366..fcb6f82d3 100644--- a/flaml/automl/task/factory.py+++ b/flaml/automl/task/factory.py@@ -1,8 +1,8 @@from typing import Optional, Unionimport numpy as np-from flaml.automl.task.generic_task import GenericTask-from flaml.automl.task.task import Task+from flaml.automl.data import DataFrame, Series+from flaml.automl.task.task import Task, TS_FORECASTdef task_factory(@@ -10,4 +10,10 @@ def task_factory(X_train: Optional[Union[np.ndarray, DataFrame]] = None,y_train: Optional[Union[np.ndarray, DataFrame, Series]] = None,) -> Task:- return GenericTask(task_name, X_train, y_train)+ from flaml.automl.task.generic_task import GenericTask+ from flaml.automl.task.time_series_task import TimeSeriesTask++ if task_name in TS_FORECAST:+ return TimeSeriesTask(task_name, X_train, y_train)+ else:+ return GenericTask(task_name, X_train, y_train)diff --git a/flaml/automl/task/generic_task.py b/flaml/automl/task/generic_task.pyindex 7f480365b..d4c83ef86 100644--- a/flaml/automl/task/generic_task.py+++ b/flaml/automl/task/generic_task.py@@ -3,21 +3,8 @@ import timefrom typing import List, Optionalimport numpy as npfrom flaml.automl.data import TS_TIMESTAMP_COL, concat-from flaml.automl.ml import EstimatorSubclass, default_cv_score_agg_func, get_val_loss-from flaml.automl.model import (- XGBoostSklearnEstimator,- XGBoostLimitDepthEstimator,- RandomForestEstimator,- LGBMEstimator,- LRL1Classifier,- LRL2Classifier,- CatBoostEstimator,- ExtraTreesEstimator,- KNeighborsEstimator,- TransformersEstimator,- TransformersEstimatorModelSelection,- SparkLGBMEstimator,-)+from flaml.automl.ml import EstimatorSubclass, get_val_loss, default_cv_score_agg_func+from flaml.automl.task.task import (Task,get_classification_objective,@@ -58,20 +45,40 @@ logger = logging.getLogger(__name__)class GenericTask(Task):- estimators = {- "xgboost": XGBoostSklearnEstimator,- "xgb_limitdepth": XGBoostLimitDepthEstimator,- "rf": RandomForestEstimator,- "lgbm": LGBMEstimator,- "lrl1": LRL1Classifier,- "lrl2": LRL2Classifier,- "catboost": CatBoostEstimator,- "extra_tree": ExtraTreesEstimator,- "kneighbor": KNeighborsEstimator,- "transformer": TransformersEstimator,- "transformer_ms": TransformersEstimatorModelSelection,- "lgbm_spark": SparkLGBMEstimator,- }+ @property+ def estimators(self):+ if self._estimators is None:+ # put this into a function to avoid circular dependency+ from flaml.automl.model import (+ XGBoostSklearnEstimator,+ XGBoostLimitDepthEstimator,+ RandomForestEstimator,+ LGBMEstimator,+ LRL1Classifier,+ LRL2Classifier,+ CatBoostEstimator,+ ExtraTreesEstimator,+ KNeighborsEstimator,+ TransformersEstimator,+ TransformersEstimatorModelSelection,+ SparkLGBMEstimator,+ )++ self._estimators = {+ "xgboost": XGBoostSklearnEstimator,+ "xgb_limitdepth": XGBoostLimitDepthEstimator,+ "rf": RandomForestEstimator,+ "lgbm": LGBMEstimator,+ "lgbm_spark": SparkLGBMEstimator,+ "lrl1": LRL1Classifier,+ "lrl2": LRL2Classifier,+ "catboost": CatBoostEstimator,+ "extra_tree": ExtraTreesEstimator,+ "kneighbor": KNeighborsEstimator,+ "transformer": TransformersEstimator,+ "transformer_ms": TransformersEstimatorModelSelection,+ }+ return self._estimatorsdef validate_data(self,@@ -233,55 +240,7 @@ class GenericTask(Task):state.groups_val = groups_valstate.groups = groups- @staticmethod- def _validate_ts_data(- dataframe,- y_train_all=None,- ):- assert (- dataframe[dataframe.columns[0]].dtype.name == "datetime64[ns]"- ), f"For '{TS_FORECAST}' task, the first column must contain timestamp values."- if y_train_all is not None:- if isinstance(y_train_all, pd.Series):- y_df = pd.DataFrame(y_train_all)- elif isinstance(y_train_all, np.ndarray):- y_df = pd.DataFrame(y_train_all, columns=["labels"])- elif isinstance(y_train_all, (psDataFrame, psSeries)):- # TODO: optimize this- set_option("compute.ops_on_diff_frames", True)- y_df = y_train_all- dataframe = dataframe.join(y_df)- duplicates = dataframe.duplicated()- if isinstance(dataframe, psDataFrame):- if duplicates.any():- logger.warning("Duplicate timestamp values found in timestamp column.")- dataframe = dataframe.drop_duplicates()- logger.warning("Removed duplicate rows based on all columns")- assert (- dataframe[[dataframe.columns[0]]].duplicated().any() is False- ), "Duplicate timestamp values with different values for other columns."- ts_series = ps.to_datetime(dataframe[dataframe.columns[0]])- inferred_freq = None # TODO: `pd.infer_freq()` is not implemented yet.- else:- if any(duplicates):- logger.warning(- "Duplicate timestamp values found in timestamp column. "- f"\n{dataframe.loc[duplicates, dataframe][dataframe.columns[0]]}"- )- dataframe = dataframe.drop_duplicates()- logger.warning("Removed duplicate rows based on all columns")- assert (- dataframe[[dataframe.columns[0]]].duplicated() is None- ), "Duplicate timestamp values with different values for other columns."- ts_series = pd.to_datetime(dataframe[dataframe.columns[0]])- inferred_freq = pd.infer_freq(ts_series)- if inferred_freq is None:- logger.warning(- "Missing timestamps detected. To avoid error with estimators, set estimator list to ['prophet']. "- )- if y_train_all is not None:- return dataframe.iloc[:, :-1], dataframe.iloc[:, -1]- return dataframe+ automl.data_size_full = len(automl._y_train_all)@staticmethoddef _split_pyspark(state, X_train_all, y_train_all, split_ratio, stratify=None):@@ -469,85 +428,50 @@ class GenericTask(Task):X_train, y_train = X_train_all, y_train_allstate.groups_all = state.groupsif X_val is None and eval_method == "holdout":- # if eval_method = holdout, make holdout dataif split_type == "time":- if self.is_ts_forecast():- period = state.fit_kwargs[- "period"- ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator- if self.is_ts_forecastpanel():- X_train_all["time_idx"] -= X_train_all["time_idx"].min()- X_train_all["time_idx"] = X_train_all["time_idx"].astype("int")- ids = state.fit_kwargs["group_ids"].copy()- ids.append(TS_TIMESTAMP_COL)- ids.append("time_idx")- y_train_all = (- pd.DataFrame(y_train_all)- if not is_spark_dataframe- else ps.DataFrame(y_train_all)- if isinstance(y_train_all, psSeries)- else y_train_all- )- y_train_all[ids] = X_train_all[ids]- X_train_all = X_train_all.sort_values(ids)- y_train_all = y_train_all.sort_values(ids)- training_cutoff = X_train_all["time_idx"].max() - period- X_train = X_train_all[X_train_all["time_idx"] <= training_cutoff]- y_train = y_train_all[y_train_all["time_idx"] <= training_cutoff].drop(columns=ids)- X_val = X_train_all[X_train_all["time_idx"] > training_cutoff]- y_val = y_train_all[y_train_all["time_idx"] > training_cutoff].drop(columns=ids)- else:- num_samples = X_train_all.shape[0]- assert period < num_samples, f"period={period}>#examples={num_samples}"- split_idx = num_samples - period- X_train = X_train_all[:split_idx]- y_train = y_train_all[:split_idx]- X_val = X_train_all[split_idx:]- y_val = y_train_all[split_idx:]+ assert not self.is_ts_forecast(), "For a TS forecast task, this code should never be called"++ is_sample_weight = "sample_weight" in state.fit_kwargs+ if not is_spark_dataframe and is_sample_weight:+ (+ X_train,+ X_val,+ y_train,+ y_val,+ state.fit_kwargs[+ "sample_weight"+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator+ state.weight_val,+ ) = train_test_split(+ X_train_all,+ y_train_all,+ state.fit_kwargs[+ "sample_weight"+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator+ test_size=split_ratio,+ shuffle=False,+ )+ elif not is_spark_dataframe and not is_sample_weight:+ X_train, X_val, y_train, y_val = train_test_split(+ X_train_all,+ y_train_all,+ test_size=split_ratio,+ shuffle=False,+ )+ elif is_spark_dataframe and is_sample_weight:+ (+ X_train,+ X_val,+ y_train,+ y_val,+ state.fit_kwargs[+ "sample_weight"+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator+ state.weight_val,+ ) = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)else:- is_sample_weight = "sample_weight" in state.fit_kwargs- if not is_spark_dataframe and is_sample_weight:- (- X_train,- X_val,- y_train,- y_val,- state.fit_kwargs[- "sample_weight"- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator- state.weight_val,- ) = train_test_split(- X_train_all,- y_train_all,- state.fit_kwargs[- "sample_weight"- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator- test_size=split_ratio,- shuffle=False,- )- elif not is_spark_dataframe and not is_sample_weight:- X_train, X_val, y_train, y_val = train_test_split(- X_train_all,- y_train_all,- test_size=split_ratio,- shuffle=False,- )- elif is_spark_dataframe and is_sample_weight:- (- X_train,- X_val,- y_train,- y_val,- state.fit_kwargs[- "sample_weight"- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator- state.weight_val,- ) = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)- else:- X_train, X_val, y_train, y_val = self._split_pyspark(- state, X_train_all, y_train_all, split_ratio- )- elif split_type == "group":+ X_train, X_val, y_train, y_val = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)+ if split_type == "group":gss = GroupShuffleSplit(n_splits=1, test_size=split_ratio, random_state=RANDOM_SEED)for train_idx, val_idx in gss.split(X_train_all, y_train_all, state.groups_all):if data_is_df:@@ -591,6 +515,7 @@ class GenericTask(Task):state, X_train_all, y_train_all, split_ratio=split_ratio)state.data_size = X_train.shape+ state.data_size_full = len(y_train_all)state.X_train, state.y_train = X_train, y_trainstate.X_val, state.y_val = X_val, y_valstate.X_train_all = X_train_all@@ -636,6 +561,7 @@ class GenericTask(Task):state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period * n_groups)else:state.kf = TimeSeriesSplit(n_splits=n_splits)+ # state.kf = TimeSeriesSplit(n_splits=n_splits)elif isinstance(split_type, str):# logger.info("Using RepeatedKFold")state.kf = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED)@@ -653,6 +579,7 @@ class GenericTask(Task):fit_kwargs,groups=None,) -> str:+ assert not self.is_ts_forecast(), "This function should never be called as part of a time-series task."if self.name == "classification":self.name = get_classification_objective(len_labels(y_train_all))if not isinstance(split_type, str):@@ -664,20 +591,6 @@ class GenericTask(Task):), "GroupKFold requires groups to be provided."return split_type- elif self.is_ts_forecast():- assert split_type in ["auto", "time"]- assert isinstance(- fit_kwargs.get("period"),- int, # NOTE: _decide_split_type is before kwargs is updated to fit_kwargs_by_estimator- ), f"missing a required integer 'period' for '{TS_FORECAST}' task."- if fit_kwargs.get("group_ids"):- # TODO (MARK) This will likely not play well with the task class- self.name = TS_FORECASTPANEL- assert isinstance(- fit_kwargs.get("group_ids"), list- ), f"missing a required List[str] 'group_ids' for '{TS_FORECASTPANEL}' task."- return "time"-elif self.is_classification():assert split_type in ["auto", "stratified", "uniform", "time", "group"]return split_type if split_type != "auto" else groups is None and "stratified" or "group"@@ -915,23 +828,23 @@ class GenericTask(Task):"xgb_limitdepth","lgbm_spark",]- if self.is_ts_forecast():- # catboost is removed because it has a `name` parameter, making it incompatible with hcrystalball- if "catboost" in estimator_list:- estimator_list.remove("catboost")- if self.is_ts_forecastregression():- try:- import prophet-- estimator_list += [- "prophet",- "arima",- "sarimax",- "holt-winters",- ]- except ImportError:- estimator_list += ["arima", "sarimax", "holt-winters"]- elif not self.is_regression():+ # if self.is_ts_forecast():+ # # catboost is removed because it has a `name` parameter, making it incompatible with hcrystalball+ # if "catboost" in estimator_list:+ # estimator_list.remove("catboost")+ # if self.is_ts_forecastregression():+ # try:+ # import prophet+ #+ # estimator_list += [+ # "prophet",+ # "arima",+ # "sarimax",+ # "holt-winters",+ # ]+ # except ImportError:+ # estimator_list += ["arima", "sarimax", "holt-winters"]+ if not self.is_regression():estimator_list += ["lrl1"]estimator_list = [@@ -961,3 +874,7 @@ class GenericTask(Task):return "ndcg"else:return "r2"++ @staticmethod+ def prepare_sample_train_data(automlstate, sample_size):+ return automlstate.prepare_sample_train_data(sample_size)diff --git a/flaml/automl/task/task.py b/flaml/automl/task/task.pyindex c833627b0..4b982492c 100644--- a/flaml/automl/task/task.py+++ b/flaml/automl/task/task.py@@ -69,8 +69,6 @@ class Task(ABC):operations which vary between tasks."""- estimators = {}-def __init__(self,task_name: str,@@ -88,6 +86,7 @@ class Task(ABC):such as in binary vs multilabel classification."""self.name = task_name+ self._estimators = Nonedef __str__(self) -> str:"""Name of this task type."""@@ -326,8 +325,7 @@ class Task(ABC):"""For backward compatibility with all the string comparisons to task"""return self.name == other- @classmethod- def estimator_class_from_str(cls, estimator_name: str) -> "flaml.automl.ml.BaseEstimator":+ def estimator_class_from_str(self, estimator_name: str) -> "flaml.automl.ml.BaseEstimator":"""Determine the estimator class corresponding to the provided name.Args:@@ -339,11 +337,11 @@ class Task(ABC):Raises:ValueError: The provided estimator_name has not been registered for this task type."""- if estimator_name in cls.estimators:- return cls.estimators[estimator_name]+ if estimator_name in self.estimators:+ return self.estimators[estimator_name]else:raise ValueError(f"{estimator_name} is not a built-in learner for this task type, "- f"only {list(cls.estimators.keys())} are supported."+ f"only {list(self.estimators.keys())} are supported.""Please use AutoML.add_learner() to add a customized learner.")diff --git a/flaml/automl/task/time_series_task.py b/flaml/automl/task/time_series_task.pynew file mode 100644index 000000000..183f6c406--- /dev/null+++ b/flaml/automl/task/time_series_task.py@@ -0,0 +1,523 @@+import logging+import time+from typing import List++import pandas as pd+import numpy as np+from scipy.sparse import issparse+from sklearn.model_selection import (+ GroupKFold,+ TimeSeriesSplit,+)++from flaml.automl.ml import get_val_loss, default_cv_score_agg_func+from flaml.automl.time_series.ts_data import (+ TimeSeriesDataset,+ DataTransformerTS,+ normalize_ts_data,+)++from flaml.automl.task.task import (+ Task,+ get_classification_objective,+ TS_FORECAST,+ TS_FORECASTPANEL,+)++logger = logging.getLogger(__name__)+++class TimeSeriesTask(Task):+ @property+ def estimators(self):+ if self._estimators is None:+ # put this into a function to avoid circular dependency+ from flaml.automl.time_series import (+ XGBoost_TS,+ XGBoostLimitDepth_TS,+ RF_TS,+ LGBM_TS,+ ExtraTrees_TS,+ CatBoost_TS,+ Prophet,+ Orbit,+ ARIMA,+ SARIMAX,+ TemporalFusionTransformerEstimator,+ HoltWinters,+ )++ self._estimators = {+ "xgboost": XGBoost_TS,+ "xgb_limitdepth": XGBoostLimitDepth_TS,+ "rf": RF_TS,+ "lgbm": LGBM_TS,+ "extra_tree": ExtraTrees_TS,+ "arima": ARIMA,+ "sarimax": SARIMAX,+ "holt-winters": HoltWinters,+ "catboost": CatBoost_TS,+ "tft": TemporalFusionTransformerEstimator,+ }++ try:+ from prophet import Prophet as foo++ self._estimators["prophet"] = Prophet+ except ImportError:+ logger.info("Couldn't import Prophet, skipping")++ try:+ from orbit.models import DLT++ self._estimators["orbit"] = Orbit+ except ImportError:+ logger.info("Couldn't import Prophet, skipping")++ return self._estimators++ # processed+ def validate_data(+ self,+ automl,+ state,+ X_train_all,+ y_train_all,+ dataframe,+ label,+ X_val=None,+ y_val=None,+ groups_val=None,+ groups=None,+ ):+ # first beat the data into a TimeSeriesDataset shape+ if isinstance(X_train_all, TimeSeriesDataset):+ # in this case, we're most likely being called by another FLAML instance+ # so all the preliminary cleaning has already been done+ pre_data = X_train_all+ val_len = len(pre_data.X_val)+ else:+ if label is None and dataframe is not None:+ raise ValueError("If data is specified via dataframe parameter, you must also specify label")++ if isinstance(y_train_all, pd.Series):+ label = y_train_all.name+ elif isinstance(y_train_all, np.ndarray):+ label = "y" # Prophet convention++ if isinstance(label, str):+ target_names = [label]+ else:+ target_names = label++ if self.time_col is None:+ if isinstance(X_train_all, pd.DataFrame):+ assert dataframe is None, "One of dataframe and X arguments must be None"+ self.time_col = X_train_all.columns[0]+ elif dataframe is not None:+ assert X_train_all is None, "One of dataframe and X arguments must be None"+ self.time_col = dataframe.columns[0]+ else:+ self.time_col = "ds"++ automl._df = True++ if X_train_all is not None:+ assert y_train_all is not None, "If X_train_all is not None, y_train_all must also be"+ assert dataframe is None, "If X_train_all is provided, dataframe must be None"+ dataframe = TimeSeriesDataset.to_dataframe(X_train_all, y_train_all, target_names, self.time_col)++ elif dataframe is not None:+ assert label is not None, "A label or list of labels must be provided."+ assert isinstance(dataframe, pd.DataFrame), "dataframe must be a pandas DataFrame"+ assert label in dataframe.columns, f"{label} must a column name in dataframe"+ else:+ raise ValueError("Must supply either X_train_all and y_train_all, or dataframe and label")++ try:+ dataframe[self.time_col] = pd.to_datetime(dataframe[self.time_col])+ except Exception:+ raise ValueError(+ f"For '{TS_FORECAST}' task, time column {self.time_col} must contain timestamp values."+ )++ dataframe = remove_ts_duplicates(dataframe, self.time_col)++ if X_val is not None:+ assert y_val is not None, "If X_val is not None, y_val must also be"+ val_df = TimeSeriesDataset.to_dataframe(X_val, y_val, target_names, self.time_col)+ val_len = len(val_df)+ else:+ val_len = 0+ val_df = None++ pre_data = TimeSeriesDataset(+ train_data=dataframe,+ time_col=self.time_col,+ target_names=target_names,+ test_data=val_df,+ )++ # TODO: should the transformer be a property of the dataset instead?+ automl._transformer = DataTransformerTS(self.time_col, label)+ Xt, yt = automl._transformer.fit_transform(pre_data.X_all, pre_data.y_all)++ df_t = pd.concat([Xt, yt], axis=1)++ data = TimeSeriesDataset(+ train_data=df_t,+ time_col=pre_data.time_col,+ target_names=pre_data.target_names,+ ).move_validation_boundary(-val_len)++ # now setup the properties of all the other relevant objects++ # TODO: where are these used? Replace with pointers to data?+ automl._X_train_all, automl._y_train_all = Xt, yt++ # TODO: where are these used?+ automl._nrow, automl._ndim = data.X_train.shape++ # make a property instead? Or just fix the call?+ automl._label_transformer = automl._transformer.label_transformer++ automl._feature_names_in_ = (+ automl._X_train_all.columns.to_list() if hasattr(automl._X_train_all, "columns") else None+ )++ self.time_col = data.time_col+ self.target_names = data.target_names++ automl._state.X_val = data+ automl._state.X_train = data+ automl._state.y_train = None+ automl._state.y_val = None+ if data.test_data is not None and len(data.test_data) > 0:+ automl._state.X_train_all = data.move_validation_boundary(len(data.test_data))+ else:+ automl._state.X_train_all = data+ automl._state.y_train_all = None++ automl._state.data_size = data.train_data.shape+ automl.data_size_full = len(data.all_data)+ automl._state.groups = None+ automl._sample_weight_full = None++ def prepare_data(+ self,+ state,+ X_train_all,+ y_train_all,+ auto_argument,+ eval_method,+ split_type,+ split_ratio,+ n_splits,+ data_is_df,+ sample_weight_full,+ time_col=None,+ ):+ state.kf = None+ state.data_size_full = len(y_train_all)++ if split_type in ["uniform", "stratified"]:+ raise ValueError(f"Split type {split_type} is not valid for time series")++ state.groups = None+ state.groups_all = None+ state.groups_val = None++ ts_data = state.X_val+ no_test_data = ts_data is None or ts_data.test_data is None or len(ts_data.test_data) == 0+ if no_test_data and eval_method == "holdout":+ # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator+ period = state.fit_kwargs["period"]++ if self.name == TS_FORECASTPANEL:+ # TODO: move this into the TimeSeriesDataset class+ X_train_all = ts_data.X_train+ y_train_all = ts_data.y_train++ X_train_all["time_idx"] -= X_train_all["time_idx"].min()+ X_train_all["time_idx"] = X_train_all["time_idx"].astype("int")+ ids = state.fit_kwargs["group_ids"].copy()+ ids.append(ts_data.time_col)+ ids.append("time_idx")+ y_train_all = pd.DataFrame(y_train_all)+ y_train_all[ids] = X_train_all[ids]+ X_train_all = X_train_all.sort_values(ids)+ y_train_all = y_train_all.sort_values(ids)+ training_cutoff = X_train_all["time_idx"].max() - period+ X_train = X_train_all[lambda x: x.time_idx <= training_cutoff]+ y_train = y_train_all[lambda x: x.time_idx <= training_cutoff].drop(columns=ids)+ X_val = X_train_all[lambda x: x.time_idx > training_cutoff]+ y_val = y_train_all[lambda x: x.time_idx > training_cutoff].drop(columns=ids)++ train_data = normalize_ts_data(+ X_train,+ ts_data.target_names,+ ts_data.time_col,+ y_train,+ )+ test_data = normalize_ts_data(+ X_val,+ ts_data.target_names,+ ts_data.time_col,+ y_val,+ )+ ts_data = TimeSeriesDataset(+ train_data,+ ts_data.time_col,+ ts_data.target_names,+ ts_data.frequency,+ test_data,+ )+ state.X_val = ts_data+ state.X_train = ts_data++ else:+ # if eval_method = holdout, make holdout data+ num_samples = ts_data.train_data.shape[0]+ assert period < num_samples, f"period={period}>#examples={num_samples}"+ state.X_val = ts_data.move_validation_boundary(-period)+ state.X_train = state.X_val++ if eval_method != "holdout":+ if self.name != TS_FORECASTPANEL:+ period = state.fit_kwargs[+ "period"+ ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator+ step_size = state.fit_kwargs.get("cv_step_size", period)++ ts_data = state.X_train+ if n_splits * step_size + 2 * period > ts_data.y_train.size:+ n_splits = int((ts_data.y_train.size - 2 * period) / step_size)+ assert n_splits >= 2, (+ f"cross validation for forecasting period={period}"+ f" requires input data with at least {2*period + 2*step_size} examples."+ )+ logger.info(f"Using nsplits={n_splits} due to data size limit.")+ state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period)+ state.kf.step_size = step_size++ else:+ n_groups = ts_data.X_train.groupby(state.fit_kwargs.get("group_ids")).ngroups+ period = state.fit_kwargs["period"]+ state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period * n_groups)++ # TODO: move task detection to Task.__init__!+ def decide_split_type(+ self,+ split_type,+ y_train_all,+ fit_kwargs,+ groups=None,+ ) -> str:+ # TODO: move into task creation!!!+ if self.name == "classification":+ self.name = get_classification_objective(len(np.unique(y_train_all)))++ # TODO: do we need this?+ if not isinstance(split_type, str):+ assert hasattr(split_type, "split") and hasattr(+ split_type, "get_n_splits"+ ), "split_type must be a string or a splitter object with split and get_n_splits methods."+ assert (+ not isinstance(split_type, GroupKFold) or groups is not None+ ), "GroupKFold requires groups to be provided."+ return split_type++ else:+ assert split_type in ["auto", "time"]+ assert isinstance(+ fit_kwargs.get("period"),+ int, # NOTE: _decide_split_type is before kwargs is updated to fit_kwargs_by_estimator+ ), f"missing a required integer 'period' for '{TS_FORECAST}' task."+ if fit_kwargs.get("group_ids"):+ # TODO (MARK) This will likely not play well with the task class+ self.name = TS_FORECASTPANEL+ assert isinstance(+ fit_kwargs.get("group_ids"), list+ ), f"missing a required List[str] 'group_ids' for '{TS_FORECASTPANEL}' task."+ return "time"++ # TODO: merge with preprocess() below+ def _preprocess(self, X, transformer=None):+ if isinstance(X, List):+ try:+ if isinstance(X[0], List):+ X = [x for x in zip(*X)]+ X = pd.DataFrame(+ dict(+ [+ (transformer._str_columns[idx], X[idx])+ if isinstance(X[0], List)+ else (transformer._str_columns[idx], [X[idx]])+ for idx in range(len(X))+ ]+ )+ )+ except IndexError:+ raise IndexError("Test data contains more columns than training data, exiting")+ elif isinstance(X, int):+ return X+ elif issparse(X):+ X = X.tocsr()+ if self.is_ts_forecast():+ X = pd.DataFrame(X)+ if transformer:+ X = transformer.transform(X)+ return X++ def preprocess(self, X, transformer=None):+ if isinstance(X, pd.DataFrame) or isinstance(X, np.ndarray) or isinstance(X, pd.Series):+ X = X.copy()+ X = normalize_ts_data(X, self.target_names, self.time_col)+ return self._preprocess(X, transformer)+ elif isinstance(X, int):+ return X+ else:+ raise ValueError(f"unknown type of X, {X.__class__}")++ def evaluate_model_CV(+ self,+ config,+ estimator,+ X_train_all,+ y_train_all,+ budget,+ kf,+ eval_metric,+ best_val_loss,+ cv_score_agg_func=None,+ log_training_metric=False,+ fit_kwargs={},+ free_mem_ratio=0, # what is this for?+ ):+ if cv_score_agg_func is None:+ cv_score_agg_func = default_cv_score_agg_func+ start_time = time.time()+ val_loss_folds = []+ log_metric_folds = []+ metric = None+ train_time = pred_time = 0+ total_fold_num = 0+ n = kf.get_n_splits()+ if self.is_classification():+ labels = np.unique(y_train_all)+ else:+ labels = fit_kwargs.get("label_list") # pass the label list on to compute the evaluation metric+ ts_data = X_train_all+ budget_per_train = budget / n+ ts_data = X_train_all+ for data in ts_data.cv_train_val_sets(kf.n_splits, kf.test_size, kf.step_size):+ estimator.cleanup()+ val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(+ config,+ estimator,+ X_train=data,+ y_train=None,+ X_val=data,+ y_val=None,+ eval_metric=eval_metric,+ labels=labels,+ budget=budget_per_train,+ log_training_metric=log_training_metric,+ fit_kwargs=fit_kwargs,+ task=self,+ weight_val=None,+ groups_val=None,+ free_mem_ratio=free_mem_ratio,+ )+ if isinstance(metric_i, dict) and "intermediate_results" in metric_i:+ del metric_i["intermediate_results"]+ total_fold_num += 1+ val_loss_folds.append(val_loss_i)+ log_metric_folds.append(metric_i)+ train_time += train_time_i+ pred_time += pred_time_i+ if time.time() - start_time >= budget:+ break+ val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)+ n = total_fold_num+ pred_time /= n+ return val_loss, metric, train_time, pred_time++ def default_estimator_list(self, estimator_list: List[str], is_spark_dataframe: bool) -> List[str]:+ assert not is_spark_dataframe, "Spark is not yet supported for time series"++ # TODO: why not do this if/then in the calling function?+ if "auto" != estimator_list:+ return estimator_list++ if self.is_ts_forecastpanel():+ return ["tft"]++ estimator_list = [+ "lgbm",+ "rf",+ "xgboost",+ "extra_tree",+ "xgb_limitdepth",+ ]++ # Catboost appears to be way slower than the others, don't include it by default+ # try:+ # import catboost+ #+ # estimator_list.append("catboost")+ # except ImportError:+ # pass++ if self.is_regression():+ estimator_list += ["arima", "sarimax"]++ try:+ import prophet++ estimator_list.append("prophet")+ except ImportError:+ pass++ return estimator_list++ def default_metric(self, metric: str) -> str:+ assert self.is_ts_forecast(), "If this is not a TS forecasting task, this code should never have been called"+ if metric == "auto":+ return "mape"+ else:+ return metric++ @staticmethod+ def prepare_sample_train_data(automlstate, sample_size):+ # we take the tail, rather than the head, for compatibility with time series++ shift = sample_size - len(automlstate.X_train.train_data)+ sampled_X_train = automlstate.X_train.move_validation_boundary(shift)++ return sampled_X_train, None, None, None+++def remove_ts_duplicates(+ X,+ time_col,+):+ """+ Assumes the targets are included+ @param X:+ @param time_col:+ @param y:+ @return:+ """++ duplicates = X.duplicated()++ if any(duplicates):+ logger.warning("Duplicate timestamp values found in timestamp column. " f"\n{X.loc[duplicates, X][time_col]}")+ X = X.drop_duplicates()+ logger.warning("Removed duplicate rows based on all columns")+ assert (+ X[[X.columns[0]]].duplicated() is None+ ), "Duplicate timestamp values with different values for other columns."++ return Xdiff --git a/flaml/automl/time_series/__init__.py b/flaml/automl/time_series/__init__.pynew file mode 100644index 000000000..0cf1c1c87--- /dev/null+++ b/flaml/automl/time_series/__init__.py@@ -0,0 +1,17 @@+from .ts_model import (+ Prophet,+ Orbit,+ ARIMA,+ SARIMAX,+ HoltWinters,+ LGBM_TS,+ XGBoost_TS,+ RF_TS,+ ExtraTrees_TS,+ XGBoostLimitDepth_TS,+ CatBoost_TS,+ TimeSeriesEstimator,+)+from .tft import TemporalFusionTransformerEstimator++from .ts_data import TimeSeriesDatasetdiff --git a/flaml/automl/time_series/feature.py b/flaml/automl/time_series/feature.pynew file mode 100644index 000000000..8cf6eb430--- /dev/null+++ b/flaml/automl/time_series/feature.py@@ -0,0 +1,34 @@+import math+import datetime+from functools import lru_cache++import pandas as pd+++def monthly_fourier_features(timestamps: pd.Series, month_fourier_degree: int = 2):+ if len(timestamps):+ data = pd.DataFrame({"time": timestamps})+ month_pos = timestamps.apply(lambda x: position_in_month(datetime.date(x.year, x.month, x.day)))+ for d in range(month_fourier_degree):+ data[f"cos{d+1}"] = (2 * (d + 1) * math.pi * month_pos).apply(math.cos)+ data[f"sin{d + 1}"] = (2 * (d + 1) * math.pi * month_pos).apply(math.sin)++ drop_cols = ["time"]+ data = data.drop(columns=drop_cols)+ return data+ else:+ columns = []+ for d in range(month_fourier_degree):+ columns += [f"cos{d+1}", f"sin{d + 1}"]++ return pd.DataFrame(columns=columns)+++@lru_cache(maxsize=4096)+def position_in_month(d: datetime.date):+ prev = datetime.date(d.year, d.month, 1) - datetime.timedelta(days=1)+ nxt = datetime.date(+ d.year + 1 if d.month == 12 else d.year, 1 if d.month == 12 else d.month + 1, 1+ ) - datetime.timedelta(days=1)+ delta = (d - prev).days / (nxt - prev).days+ return deltadiff --git a/flaml/automl/time_series/sklearn.py b/flaml/automl/time_series/sklearn.pynew file mode 100644index 000000000..175cef848--- /dev/null+++ b/flaml/automl/time_series/sklearn.py@@ -0,0 +1,156 @@+try:+ import pandas as pd+ from pandas import DataFrame, Series, to_datetime+except ImportError:++ class PD:+ pass++ pd = PD()+ pd.DataFrame = None+ pd.Series = None+ DataFrame = Series = None++import numpy as np+from sklearn.preprocessing import StandardScaler+from sklearn.decomposition import PCA+++def make_lag_features(X: pd.DataFrame, y: pd.Series, lags: int):+ """Transform input data X, y into autoregressive form - shift+ them appropriately based on horizon and create `lags` columns.++ Parameters+ ----------+ X : pandas.DataFrame+ Input features.++ y : array_like, (1d)+ Target vector.++ horizon : int+ length of X for `predict` method++ Returns+ -------+ pandas.DataFrame+ shifted dataframe with `lags` columns+ """+ lag_features = []++ # make sure we show y's _previous_ value to exclude data leaks+ X = X.reset_index(drop=True)+ X["lag_" + y.name] = y.shift(1).values++ X_lag = X.copy()+ for i in range(0, lags):+ X_lag.columns = [f"{c}_lag_{i}" for c in X.columns]+ lag_features.append(X_lag)+ X_lag = X_lag.shift(1)++ X_lags = pd.concat(lag_features, axis=1)+ X_out = X_lags.dropna().reset_index(drop=True)+ assert len(X_out) + lags == len(X)+ return X_out+++class SklearnWrapper:+ def __init__(+ self,+ model_class: type,+ horizon: int,+ lags: int,+ init_params: dict = None,+ fit_params: dict = None,+ pca_features: bool = False,+ ):+ init_params = init_params if init_params else {}+ self.fit_params = fit_params if fit_params else {}+ self.lags = lags+ self.horizon = horizon+ # TODO: use multiregression where available+ self.models = [model_class(**init_params) for _ in range(horizon)]+ self.pca_features = pca_features+ if self.pca_features:+ self.norm = StandardScaler()+ self.pca = None++ def fit(self, X: pd.DataFrame, y: pd.Series, **kwargs):+ self._X = X+ self._y = y++ fit_params = {**self.fit_params, **kwargs}+ X_feat = make_lag_features(X, y, self.lags)+ if self.pca_features:+ X_trans = self.norm.fit_transform(X_feat)++ cum_expl_var = np.cumsum(PCA(svd_solver="full").fit(X_trans).explained_variance_ratio_)+ self.pca = PCA(svd_solver="full", n_components=np.argmax(1 - cum_expl_var < 1e-6))+ X_trans = self.pca.fit_transform(X_trans)+ else:+ X_trans = X_feat++ for i, model in enumerate(self.models):+ offset = i + self.lags+ model.fit(X_trans[: len(X) - offset], y[offset:], **fit_params)+ return self++ def predict(self, X, X_train=None, y_train=None):+ if X_train is None:+ X_train = self._X+ if y_train is None:+ y_train = self._y++ X_train = X_train.reset_index(drop=True)+ X_train[self._y.name] = y_train.values+ Xall = pd.concat([X_train, X], axis=0).reset_index(drop=True)+ y = Xall.pop(self._y.name)++ X_feat = make_lag_features(Xall[: len(X_train) + 1], y[: len(X_train) + 1], self.lags)+ if self.pca_features:+ X_trans = self.pca.transform(self.norm.transform(X_feat))+ else:+ X_trans = X_feat+ # predict all horizons from the latest features vector+ preds = pd.Series([m.predict(X_trans[-1:])[0] for m in self.models])+ if len(preds) < len(X):+ # recursive call if len(X) > trained horizon+ y_train = pd.concat([y_train, preds], axis=0, ignore_index=True)+ preds = pd.concat(+ [+ preds,+ self.predict(+ X=Xall[len(y_train) :],+ X_train=Xall[: len(y_train)],+ y_train=y_train,+ ),+ ],+ axis=0,+ ignore_index=True,+ )+ if len(preds) > len(X):+ preds = preds[: len(X)]++ preds.index = X.index+ # TODO: do we want auto-clipping?+ # return self._clip_predictions(preds)+ return preds++ # TODO: fix+ # @staticmethod+ # def _adjust_holidays(X):+ # """Transform 'holiday' columns to binary feature.+ #+ # Parameters+ # ----------+ # X : pandas.DataFrame+ # Input features with 'holiday' column.+ #+ # Returns+ # -------+ # pandas.DataFrame+ # Holiday feature in numeric form+ # """+ # return X.assign(+ # **{col: X[col] != "" for col in X.filter(like="_holiday_").columns}+ # )diff --git a/flaml/automl/time_series/tft.py b/flaml/automl/time_series/tft.pynew file mode 100644index 000000000..11a5714d9--- /dev/null+++ b/flaml/automl/time_series/tft.py@@ -0,0 +1,183 @@+import time++try:+ import pandas as pd+ from pandas import DataFrame, Series, to_datetime+except ImportError:++ class PD:+ pass++ pd = PD()+ pd.DataFrame = None+ pd.Series = None+ DataFrame = Series = None++from flaml import tune+from flaml.automl.data import add_time_idx_col+from flaml.automl.time_series.ts_data import TimeSeriesDataset+from flaml.automl.time_series.ts_model import TimeSeriesEstimator+++class TemporalFusionTransformerEstimator(TimeSeriesEstimator):+ """The class for tuning Temporal Fusion Transformer"""++ @classmethod+ def search_space(cls, data, task, pred_horizon, **params):+ space = {+ "gradient_clip_val": {+ "domain": tune.loguniform(lower=0.01, upper=100.0),+ "init_value": 0.01,+ },+ "hidden_size": {+ "domain": tune.lograndint(lower=8, upper=512),+ "init_value": 16,+ },+ "hidden_continuous_size": {+ "domain": tune.randint(lower=1, upper=65),+ "init_value": 8,+ },+ "attention_head_size": {+ "domain": tune.randint(lower=1, upper=5),+ "init_value": 4,+ },+ "dropout": {+ "domain": tune.uniform(lower=0.1, upper=0.3),+ "init_value": 0.1,+ },+ "learning_rate": {+ "domain": tune.loguniform(lower=0.00001, upper=1.0),+ "init_value": 0.001,+ },+ }+ return space++ def transform_ds(self, X_train: TimeSeriesDataset, y_train, **kwargs):+ self.data = X_train.train_data++ max_prediction_length = kwargs["period"]+ self.max_encoder_length = kwargs["max_encoder_length"]+ training_cutoff = self.data["time_idx"].max() - max_prediction_length++ from pytorch_forecasting import TimeSeriesDataSet+ from pytorch_forecasting.data import GroupNormalizer++ self.group_ids = kwargs["group_ids"].copy()+ training = TimeSeriesDataSet(+ self.data[lambda x: x.time_idx <= training_cutoff],+ time_idx="time_idx",+ target=X_train.target_names[0],+ group_ids=self.group_ids,+ min_encoder_length=kwargs.get(+ "min_encoder_length", self.max_encoder_length // 2+ ), # keep encoder length long (as it is in the validation set)+ max_encoder_length=self.max_encoder_length,+ min_prediction_length=1,+ max_prediction_length=max_prediction_length,+ static_categoricals=kwargs.get("static_categoricals", []),+ static_reals=kwargs.get("static_reals", []),+ time_varying_known_categoricals=kwargs.get("time_varying_known_categoricals", []),+ time_varying_known_reals=kwargs.get("time_varying_known_reals", []),+ time_varying_unknown_categoricals=kwargs.get("time_varying_unknown_categoricals", []),+ time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),+ variable_groups=kwargs.get(+ "variable_groups", {}+ ), # group of categorical variables can be treated as one variable+ lags=kwargs.get("lags", {}),+ target_normalizer=GroupNormalizer(+ groups=kwargs["group_ids"], transformation="softplus"+ ), # use softplus and normalize by group+ add_relative_time_idx=True,+ add_target_scales=True,+ add_encoder_length=True,+ )++ # create validation set (predict=True) which means to predict the last max_prediction_length points in time+ # for each series+ validation = TimeSeriesDataSet.from_dataset(training, self.data, predict=True, stop_randomization=True)++ # create dataloaders for model+ batch_size = kwargs.get("batch_size", 64)+ train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)+ val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size * 10, num_workers=0)++ return training, train_dataloader, val_dataloader++ def fit(self, X_train, y_train, budget=None, **kwargs):+ import warnings+ import pytorch_lightning as pl+ import torch+ from pytorch_forecasting import TemporalFusionTransformer+ from pytorch_forecasting.metrics import QuantileLoss+ from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor+ from pytorch_lightning.loggers import TensorBoardLogger++ # a bit of monkey patching to fix the MacOS test+ # all the log_prediction method appears to do is plot stuff, which ?breaks github tests+ def log_prediction(*args, **kwargs):+ pass++ TemporalFusionTransformer.log_prediction = log_prediction++ warnings.filterwarnings("ignore")+ current_time = time.time()+ super().fit(X_train, **kwargs)+ training, train_dataloader, val_dataloader = self.transform_ds(X_train, y_train, **kwargs)+ params = self.params.copy()+ gradient_clip_val = params.pop("gradient_clip_val", None)+ params.pop("n_jobs", None)+ max_epochs = kwargs.get("max_epochs", 20)+ early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")+ lr_logger = LearningRateMonitor() # log the learning rate+ logger = TensorBoardLogger(kwargs.get("log_dir", "lightning_logs")) # logging results to a tensorboard+ default_trainer_kwargs = dict(+ gpus=self._kwargs.get("gpu_per_trial", [0]) if torch.cuda.is_available() else None,+ max_epochs=max_epochs,+ gradient_clip_val=gradient_clip_val,+ callbacks=[lr_logger, early_stop_callback],+ logger=logger,+ )+ trainer = pl.Trainer(+ **default_trainer_kwargs,+ )+ tft = TemporalFusionTransformer.from_dataset(+ training,+ **params,+ lstm_layers=2, # 2 is mostly optimal according to documentation+ output_size=7, # 7 quantiles by default+ loss=QuantileLoss(),+ log_interval=10, # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches+ reduce_on_plateau_patience=4,+ )+ # fit network+ trainer.fit(+ tft,+ train_dataloaders=train_dataloader,+ val_dataloaders=val_dataloader,+ )+ best_model_path = trainer.checkpoint_callback.best_model_path+ best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)+ train_time = time.time() - current_time+ self._model = best_tft+ return train_time++ def predict(self, X):+ ids = self.group_ids.copy()+ ids.append(self.time_col)+ encoder_data = self.data[lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length]+ # following pytorchforecasting example, make all target values equal to the last data+ last_data_cols = self.group_ids.copy()+ last_data_cols.append(self.target_names[0])+ last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]+ decoder_data = X.X_val if isinstance(X, TimeSeriesDataset) else X+ if "time_idx" not in decoder_data:+ decoder_data = add_time_idx_col(decoder_data)+ decoder_data["time_idx"] += encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()+ decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)+ decoder_data = decoder_data.sort_values(ids)+ new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)+ new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")+ new_raw_predictions = self._model.predict(new_prediction_data)+ index = [decoder_data[idx].to_numpy() for idx in ids]+ predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)+ return predictionsdiff --git a/flaml/automl/time_series/ts_data.py b/flaml/automl/time_series/ts_data.pynew file mode 100644index 000000000..2dc7922a1--- /dev/null+++ b/flaml/automl/time_series/ts_data.py@@ -0,0 +1,544 @@+import copy+import datetime+import math+from dataclasses import dataclass, field+from typing import List, Optional, Callable, Dict, Generator, Union++import numpy as np++try:+ import pandas as pd+ from pandas import DataFrame, Series, to_datetime+ from scipy.sparse import issparse+ from sklearn.preprocessing import LabelEncoder+ from sklearn.impute import SimpleImputer+ from sklearn.compose import ColumnTransformer++ from .feature import monthly_fourier_features+except ImportError:++ class PD:+ pass++ pd = PD()+ pd.DataFrame = None+ pd.Series = None+ DataFrame = Series = None+++@dataclass+class TimeSeriesDataset:+ train_data: pd.DataFrame+ time_idx: str+ time_col: str+ target_names: List[str]+ frequency: str+ test_data: pd.DataFrame+ time_varying_known_categoricals: List[str] = field(default_factory=lambda: [])+ time_varying_known_reals: List[str] = field(default_factory=lambda: [])+ time_varying_unknown_categoricals: List[str] = field(default_factory=lambda: [])+ time_varying_unknown_reals: List[str] = field(default_factory=lambda: [])++ def __init__(+ self,+ train_data: pd.DataFrame,+ time_col: str,+ target_names: Union[str, List[str]],+ time_idx: str = "time_idx",+ test_data: Optional[pd.DataFrame] = None,+ ):+ self.train_data = train_data+ self.time_col = time_col+ self.time_idx = time_idx+ self.target_names = [target_names] if isinstance(target_names, str) else list(target_names)+ assert isinstance(self.target_names, list)+ assert len(self.target_names)++ self.frequency = pd.infer_freq(train_data[time_col].unique())+ assert self.frequency is not None, "Only time series of regular frequency are currently supported."++ float_cols = list(train_data.select_dtypes(include=["floating"]).columns)+ self.time_varying_known_reals = list(set(float_cols) - set(self.target_names))++ self.time_varying_known_categoricals = list(+ set(train_data.columns) - set(self.time_varying_known_reals) - set(self.target_names) - {time_col}+ )+ if test_data is not None:+ self.test_data = test_data+ else:+ self.test_data = pd.DataFrame(columns=self.train_data.columns)++ def add_test_data(self, X: pd.DataFrame) -> "TimeSeriesDataset":+ assert self.time_col in X.columns+ train_data = self.all_data[self.all_data[self.time_col] < X[self.time_col].min()]+ return TimeSeriesDataset(train_data, self.time_col, self.target_names, self.time_idx, X)++ @staticmethod+ def to_dataframe(X, y, target_names: List[str], time_col: str):+ assert len(X) == len(y), "X_val and y_val must have the same length"+ validate_data_basic(X, y)+ # coerce them into a dataframe+ val_df = normalize_ts_data(X, target_names, time_col, y)+ return val_df++ @property+ def all_data(self):+ if len(self.test_data):+ return pd.concat([self.train_data, self.test_data], axis=0)+ else:+ return self.train_data++ @property+ def regressors(self):+ return self.time_varying_known_categoricals + self.time_varying_known_reals++ @property+ def end_date(self):+ test_len = 0 if self.test_data is None else len(self.test_data)+ data = self.test_data if test_len else self.train_data+ return data.iloc[-1][self.time_col]++ def _X(self, df: pd.DataFrame):+ features = [col for col in df.columns if col not in self.target_names]+ return df[features]++ def _y(self, df: pd.DataFrame):+ if len(self.target_names) > 1:+ return df[self.target_names]+ else:+ return df[self.target_names[0]]++ @property+ def X_train(self) -> pd.DataFrame:+ return self._X(self.train_data)++ @property+ def X_val(self) -> pd.DataFrame:+ return self._X(self.test_data)++ @property+ def X_all(self) -> pd.DataFrame:+ return pd.concat([self.X_train, self.X_val], axis=0)++ @property+ def y_train(self) -> pd.DataFrame:+ return self._y(self.train_data)++ @property+ def y_val(self) -> pd.DataFrame:+ return self._y(self.test_data)++ @property+ def y_all(self) -> pd.DataFrame:+ return self._y(self.all_data)++ def next_scale(self) -> int:+ scale_map = {"D": 7, "MS": 12}+ return scale_map.get(self.frequency, 8)++ def known_features_to_floats(self, train: bool, drop_first: bool = True) -> np.ndarray:+ # this is a bit tricky as shapes for train and test data must match, so need to encode together+ combined = pd.concat(+ [+ self.train_data,+ self.test_data,+ ],+ ignore_index=True,+ )++ cat_one_hots = pd.get_dummies(+ combined[self.time_varying_known_categoricals],+ columns=self.time_varying_known_categoricals,+ drop_first=drop_first,+ ).values.astype(float)++ reals = combined[self.time_varying_known_reals].values.astype(float)+ both = np.concatenate([reals, cat_one_hots], axis=1)++ if train:+ return both[: len(self.train_data)]+ else:+ return both[len(self.train_data) :]++ # def unique_dimension_values(self) -> np.ndarray:+ # # this is the same set for train and test data, by construction+ # return self.combine_dims(self.train_data).unique()+ #+ # def combine_dims(self, df):+ # return df.apply(lambda row: tuple([row[d] for d in self.dimensions]), axis=1)++ def to_univariate(self) -> Dict[str, "TimeSeriesDataset"]:+ """+ Convert a multivariate TrainingData to a dict of univariate ones+ @param df:+ @return:+ """++ train_dims = self.combine_dims(self.train_data)+ test_dims = self.combine_dims(self.test_data)++ out = {}+ for d in train_dims.unique():+ out[d] = copy.copy(self)+ out[d].train_data = self.train_data[train_dims == d]+ out[d].test_data = self.test_data[test_dims == d]+ return out++ def move_validation_boundary(self, steps: int) -> "TimeSeriesDataset":+ out = copy.copy(self)+ if steps > 0:+ out.train_data = pd.concat([self.train_data, self.test_data[:steps]])+ out.test_data = self.test_data[steps:]+ elif steps < 0:+ out.train_data = self.train_data[:steps]+ if len(self.test_data):+ out.test_data = pd.concat([self.train_data[steps:], self.test_data])+ else:+ out.test_data = self.train_data[steps:]++ return out++ def cv_train_val_sets(+ self, n_splits: int, val_length: int, step_size: int+ ) -> Generator["TimeSeriesDataset", None, None]:+ max_index = len(self.train_data) - 1+ for i in range(n_splits):+ out = copy.copy(self)+ val_start = max_index - (n_splits - i - 1) * step_size - val_length+ out.train_data = self.train_data[:val_start]+ out.test_data = self.train_data[val_start : val_start + val_length]+ yield out++ def filter(self, filter_fun: Callable) -> "TimeSeriesDataset":+ if filter_fun is None:+ return self+ out = copy.copy(self)+ out.train_data = self.train_data[filter_fun]+ out.test_data = self.test_data[filter_fun]+ return out++ def prettify_prediction(self, y_pred: Union[pd.DataFrame, pd.Series, np.ndarray]):+ if self.test_data is not None and len(self.test_data):+ assert len(y_pred) == len(self.test_data)++ if isinstance(y_pred, np.ndarray):+ y_pred = pd.DataFrame(data=y_pred, columns=self.target_names, index=self.test_data.index)+ elif isinstance(y_pred, pd.Series):+ assert len(self.target_names) == 1, "Not enough columns in y_pred"+ y_pred.name = self.target_names[0]+ y_pred = pd.DataFrame(y_pred)+ y_pred.index = self.test_data.index+ elif isinstance(y_pred, pd.DataFrame):+ y_pred.index = self.test_data.index++ if self.time_col not in y_pred.columns:+ y_pred[self.time_col] = self.test_data[self.time_col]++ else:+ if isinstance(y_pred, np.ndarray):+ raise ValueError("Can't enrich np.ndarray as self.test_data is None")+ elif isinstance(y_pred, pd.Series):+ assert len(self.target_names) == 1, "Not enough columns in y_pred"+ y_pred = pd.DataFrame({self.target_names[0]: y_pred})+ # TODO auto-create the timestamps for the time column instead of throwing+ raise NotImplementedError("Need a non-None test_data for this to work, for now")++ assert isinstance(y_pred, pd.DataFrame)+ assert self.time_col in y_pred.columns+ assert all([t in y_pred.columns for t in self.target_names])+ return y_pred++ def merge_prediction_with_target(self, y_pred: Union[pd.DataFrame, pd.Series, np.ndarray]):+ y_pred = self.prettify_prediction(y_pred)+ return pd.concat([self.train_data[[self.time_col] + self.target_names], y_pred], axis=0)+++def enrich_dataframe(+ df: Union[pd.DataFrame, pd.Series],+ fourier_degree: int,+ remove_constants: bool = False,+ fourier_time: bool = True,+) -> pd.DataFrame:+ if isinstance(df, pd.Series):+ df = pd.DataFrame(df)++ new_cols = []+ for col in df.columns:+ if df[col].dtype.name == "datetime64[ns]":+ extras = monthly_fourier_features(df[col], fourier_degree)+ extras.columns = [f"{col}_{c}" for c in extras.columns]+ extras.index = df.index+ new_cols.append(extras)+ date_feat = date_feature_dict_fourier(df[col]) if fourier_time else date_feature_dict(df[col])+ if remove_constants:+ re_date_feat = {k: v for k, v in date_feat.items() if v.nunique(dropna=False) >= 2}+ else:+ re_date_feat = date_feat++ date_feat = pd.DataFrame(re_date_feat, index=df.index)+ new_cols.append(date_feat)++ return pd.concat([df] + new_cols, axis=1, verify_integrity=True)+++def enrich_dataset(+ X: TimeSeriesDataset,+ fourier_degree: int = 0,+ remove_constants: bool = False,+ fourier_time: bool = True,+) -> TimeSeriesDataset:+ new_train = enrich_dataframe(X.train_data, fourier_degree, remove_constants, fourier_time)+ new_test = (+ None if X.test_data is None else enrich_dataframe(X.test_data, fourier_degree, remove_constants, fourier_time)+ )+ return TimeSeriesDataset(+ train_data=new_train,+ time_col=X.time_col,+ target_names=X.target_names,+ time_idx=X.time_idx,+ test_data=new_test,+ )+++def date_feature_dict(timestamps: pd.Series) -> dict:+ tmp_dt = timestamps.dt+ column = timestamps.name+ pre_columns_dict = {+ # f"{column}_year": tmp_dt.year, # not stationary+ f"{column}_month": tmp_dt.month,+ # f"{column}_day": tmp_dt.day,# taken care of with monthly fourier features+ f"{column}_hour": tmp_dt.hour,+ f"{column}_minute": tmp_dt.minute,+ f"{column}_second": tmp_dt.second,+ f"{column}_dayofweek": tmp_dt.dayofweek,+ f"{column}_dayofyear": tmp_dt.dayofyear,+ f"{column}_quarter": tmp_dt.quarter,+ }++ new_columns_dict = {}+ for k, v in pre_columns_dict.items():+ new_columns_dict.update(fourier_series(v, k))++ return new_columns_dict+++def date_feature_dict_fourier(timestamps: pd.Series) -> dict:+ tmp_dt = timestamps.dt+ column = timestamps.name+ pre_columns_dict = {+ # f"{column}_year": tmp_dt.year, # not stationary+ f"{column}_month": tmp_dt.month / 12.0,+ # f"{column}_day": tmp_dt.day,# taken care of with monthly fourier features+ f"{column}_hour": tmp_dt.hour / 24.0,+ f"{column}_minute": tmp_dt.minute / 60.0,+ f"{column}_second": tmp_dt.second / 60.0,+ f"{column}_dayofweek": tmp_dt.dayofweek / 7.0,+ f"{column}_dayofyear": tmp_dt.dayofyear / 366.0,+ f"{column}_quarter": tmp_dt.quarter / 4.0,+ }++ new_columns_dict = {}+ for k, v in pre_columns_dict.items():+ new_columns_dict.update(fourier_series(v, k))++ return new_columns_dict+++def fourier_series(feature: pd.Series, name: str):+ """+ Assume feature goes from 0 to 1 cyclically, transform that into Fourier+ @param feature: input feature+ @return: sin(2pi*feature), cos(2pi*feature)+ """+ return {+ name + "_sin": np.sin(2 * math.pi * feature),+ name + "_cos": np.cos(2 * math.pi * feature),+ }+++class DataTransformerTS:+ """Transform input time series training data."""++ def __init__(self, time_col: str, label: Union[str, List[str]], time_idx: str = "time_idx"):+ self.time_col = time_col+ self.time_idx = time_idx+ self.label = label+ self.cat_columns = []+ self.num_columns = []+ self.datetime_columns = []+ self.drop_columns = []++ @property+ def _drop(self):+ return len(self.drop_columns)++ def fit(self, X: Union[DataFrame, np.array], y):+ """Fit transformer.++ Args:+ X: A numpy array or a pandas dataframe of training data.+ y: A numpy array or a pandas series of labels.++ Returns:+ X: Processed numpy array or pandas dataframe of training data.+ y: Processed numpy array or pandas series of labels.+ """+ assert isinstance(X, DataFrame)+ X = X.copy()+ n = X.shape[0]++ assert len(self.num_columns) == 0, "Trying to call fit() twice, something is wrong"++ for column in X.columns:+ # sklearn/utils/validation.py needs int/float values+ if X[column].dtype.name in ("object", "category"):+ if (+ # drop columns where all values are the same+ X[column].nunique() == 1+ # this drops UID-type cols+ or X[column].nunique(dropna=True) == n - X[column].isnull().sum()+ ):+ self.drop_columns.append(column)+ elif column != self.time_idx:+ self.cat_columns.append(column)+ elif X[column].nunique(dropna=True) < 2:+ self.drop_columns.append(column)+ elif X[column].dtype.name == "datetime64[ns]":+ pass # these will be processed at model level,+ # so they can also be done in the predict method+ else:+ self.num_columns.append(column)++ if self.num_columns:+ self.transformer = ColumnTransformer(+ [+ (+ "continuous",+ SimpleImputer(missing_values=np.nan, strategy="median"),+ self.num_columns,+ )+ ]+ )++ self.transformer.fit(X[self.num_columns])+ else:+ self.transformer = None++ # TODO: revisit for multivariate series, and recast for a single df input anyway+ if isinstance(y, Series):+ y = y.rename(self.label)++ if isinstance(y, pd.DataFrame):+ ycol = y[y.columns[0]]+ elif isinstance(y, pd.Series):+ ycol = y+ else:+ raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")++ if not pd.api.types.is_numeric_dtype(ycol):+ self.label_transformer = LabelEncoder()+ self.label_transformer.fit(ycol)+ else:+ self.label_transformer = None++ def transform(self, X: Union[DataFrame, np.array], y=None):+ # TODO: revisit for multivariate series, and recast for a single df input anyway+ if self.label_transformer is not None and y is not None:+ if isinstance(y, pd.DataFrame):+ ycol = y[y.columns[0]]+ elif isinstance(y, pd.Series):+ ycol = y+ else:+ raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")+ y_tr = self.label_transformer.transform(ycol)+ y.iloc[:] = y_tr.reshape(y.shape)++ X.drop(columns=self.drop_columns, inplace=True)++ for col in self.cat_columns:+ if X[col].dtype.name == "category":+ if "__NAN__" not in X[col].cat.categories:+ X[col] = X[col].cat.add_categories("__NAN__").fillna("__NAN__")+ else:+ X[col] = X[col].fillna("__NAN__")+ X[col] = X[col].astype("category")++ for column in self.num_columns:+ X[column] = X[column].fillna(np.nan)++ if self.transformer is not None:+ X[self.num_columns] = self.transformer.transform(X[self.num_columns])++ if y is None:+ return X+ return X, y++ def fit_transform(self, X: Union[DataFrame, np.array], y):+ self.fit(X, y)+ return self.transform(X, y)+++def create_forward_frame(+ frequency: str,+ steps: int,+ test_end_date: datetime.datetime,+ time_col: str,+):+ start_date = test_end_date + pd.Timedelta(1, frequency)+ times = pd.date_range(+ start=start_date,+ periods=steps,+ freq=frequency,+ )+ return pd.DataFrame({time_col: times})+++def normalize_ts_data(X_train_all, target_names, time_col, y_train_all=None):+ if isinstance(X_train_all, TimeSeriesDataset):+ return X_train_all++ if issparse(X_train_all):+ X_train_all = X_train_all.tocsr()++ if isinstance(X_train_all, np.ndarray) and len(X_train_all.shape) == 1:+ X_train_all = np.reshape(X_train_all, (X_train_all.size, 1))++ if isinstance(X_train_all, np.ndarray):+ X_train_all = pd.DataFrame(+ X_train_all,+ columns=[time_col] + [f"x{i}" for i in range(X_train_all.shape[1] - 1)],+ )++ if y_train_all is None:+ return X_train_all+ else:+ if isinstance(y_train_all, np.ndarray):+ # TODO: will need to revisit this when doing multivariate y+ y_train_all = pd.DataFrame(+ y_train_all.reshape(len(X_train_all), -1),+ columns=target_names,+ index=X_train_all.index,+ )+ elif isinstance(y_train_all, pd.Series):+ y_train_all = pd.DataFrame(y_train_all)+ y_train_all.index = X_train_all.index++ dataframe = pd.concat([X_train_all, y_train_all], axis=1)++ return dataframe+++def validate_data_basic(X_train_all, y_train_all):+ assert isinstance(X_train_all, np.ndarray) or issparse(X_train_all) or isinstance(X_train_all, pd.DataFrame), (+ "X_train_all must be a numpy array, a pandas dataframe, " "or Scipy sparse matrix."+ )++ assert (+ isinstance(y_train_all, np.ndarray)+ or isinstance(y_train_all, pd.Series)+ or isinstance(y_train_all, pd.DataFrame)+ ), "y_train_all must be a numpy array or a pandas series or DataFrame."++ assert X_train_all.size != 0 and y_train_all.size != 0, "Input data must not be empty, use None if no data"++ assert X_train_all.shape[0] == y_train_all.shape[0], "# rows in X_train must match length of y_train."diff --git a/flaml/automl/time_series/ts_model.py b/flaml/automl/time_series/ts_model.pynew file mode 100644index 000000000..da1bfcbaf--- /dev/null+++ b/flaml/automl/time_series/ts_model.py@@ -0,0 +1,760 @@+import time+import logging+import os+from datetime import datetime+import math+from typing import List, Optional, Union++try:+ import pandas as pd+ from pandas import DataFrame, Series, to_datetime+except ImportError:++ class PD:+ pass++ pd = PD()+ pd.DataFrame = None+ pd.Series = None+ DataFrame = Series = None+++import numpy as np++from flaml import tune+from flaml.model import (+ suppress_stdout_stderr,+ SKLearnEstimator,+ logger,+ LGBMEstimator,+ XGBoostSklearnEstimator,+ RandomForestEstimator,+ ExtraTreesEstimator,+ XGBoostLimitDepthEstimator,+ CatBoostEstimator,+)+from flaml.data import TS_TIMESTAMP_COL, TS_VALUE_COL+from flaml.automl.time_series.ts_data import (+ TimeSeriesDataset,+ enrich_dataset,+ enrich_dataframe,+ normalize_ts_data,+ create_forward_frame,+)+from flaml.automl.task import Task+++class TimeSeriesEstimator(SKLearnEstimator):+ def __init__(self, task="ts_forecast", n_jobs=1, **params):+ super().__init__(task, **params)+ self.time_col: Optional[str] = None+ self.target_names: Optional[Union[str, List[str]]] = None+ self.frequency: Optional[str] = None+ self.end_date: Optional[datetime] = None+ self.regressors: Optional[List[str]] = None++ def enrich(+ self,+ X: Union[int, TimeSeriesDataset, DataFrame],+ remove_constants: bool = False,+ ):+ X = normalize_ts_data(X, None, self.time_col, None)+ if isinstance(X, int):+ X = create_forward_frame(self.frequency, X, self.end_date, self.time_col)++ fourier_degree = self.params.get("monthly_fourier_degree", 4)++ if isinstance(X, TimeSeriesDataset):+ return enrich_dataset(+ X,+ fourier_degree,+ remove_constants=remove_constants,+ fourier_time=self.params.get("fourier_time_features"),+ )++ return enrich_dataframe(+ X,+ fourier_degree,+ remove_constants=remove_constants,+ fourier_time=self.params.get("fourier_time_features"),+ )++ @classmethod+ def search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int):+ space = cls._search_space(data=data, task=task, pred_horizon=pred_horizon)+ space.update(cls.top_search_space())+ return space++ @staticmethod+ def adjust_scale(scale: int, data_len: int, pred_horizon: int):+ points = data_len - pred_horizon+ max_lags = math.floor(points / scale)++ while scale > 2:+ if max_lags >= 2:+ break+ scale = math.ceil(scale / 1.7)+ max_lags = math.floor(points / scale)++ assert scale >= 2 and max_lags >= 2, f"Too few points ({data_len}) for prediction horizon {pred_horizon}"++ return scale, max_lags++ @classmethod+ def top_search_space(cls):+ return {+ "monthly_fourier_degree": {+ "domain": tune.randint(lower=0, upper=8),+ "init_value": 4,+ "low_cost_init_value": 2,+ },+ "fourier_time_features": {+ "domain": tune.randint(lower=0, upper=2), # tune.choice([True, False]),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "pca_features": { # disable for now, will deal with occasional svd fail later+ "domain": tune.choice([False]),+ "init_value": False,+ "low_cost_init_value": False,+ },+ }++ @classmethod+ def top_level_params(cls):+ return ["monthly_fourier_degree"]++ def _join(self, X_train, y_train):+ assert TS_TIMESTAMP_COL in X_train, (+ "Dataframe for training ts_forecast model must have column"+ f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'+ )+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])+ train_df = X_train.join(y_train)+ return train_df++ def fit(self, X_train: TimeSeriesDataset, y_train=None, budget=None, **kwargs):+ # TODO purge y_train+ self.time_col = X_train.time_col+ self.target_names = X_train.target_names+ self.X_train = X_train+ self.frequency = self.X_train.frequency+ self.end_date = self.X_train.end_date++ def score(self, X_val: DataFrame, y_val: Series, **kwargs):+ from sklearn.metrics import r2_score+ from ..ml import metric_loss_score++ y_pred = self.predict(X_val, **kwargs)+ if isinstance(X_val, TimeSeriesDataset):+ y_val = X_val.test_data[X_val.target_names[0]]+ self._metric = kwargs.get("metric", None)+ if self._metric:+ return metric_loss_score(self._metric, y_pred, y_val)+ else:+ return r2_score(y_pred, y_val)+++class Orbit(TimeSeriesEstimator):+ def fit(self, X_train: TimeSeriesDataset, y_train=None, budget=None, **kwargs):+ # This may be needed to get PyStan to run, needed for Orbit+ os.environ["KMP_DUPLICATE_LIB_OK"] = "True"+ from orbit.models import DLT++ # y_train is ignored, just need it for signature compatibility with other classes+ super().fit(X_train, y_train, budget=budget, **kwargs)+ current_time = time.time()+ self.logger = logging.getLogger("orbit").setLevel(logging.WARNING)++ model_class = self.params.get("model_class", DLT)+ self._model = model_class(+ response_col=X_train.target_names[0],+ date_col=X_train.time_col,+ regressor_col=X_train.regressors,+ # TODO: infer seasonality from frequency+ **self.params,+ )++ with suppress_stdout_stderr():+ self._model.fit(df=X_train.train_data.copy())++ train_time = time.time() - current_time+ return train_time++ def predict(self, X: Union[TimeSeriesDataset, DataFrame], **kwargs):+ if isinstance(X, int):+ X = create_forward_frame(+ self.frequency,+ X,+ self.end_date,+ self.time_col,+ )++ elif isinstance(X, TimeSeriesDataset):+ data = X+ X = data.test_data[[self.time_col] + X.regressors]++ if self._model is not None:+ forecast = self._model.predict(X, **kwargs)+ out = (+ DataFrame(+ forecast[+ [+ self.time_col,+ "prediction",+ "prediction_5",+ "prediction_95",+ ]+ ]+ )+ .reset_index(drop=True)+ .rename(+ columns={+ "prediction": self.target_names[0],+ }+ )+ )++ return out+ else:+ self.logger.warning("Estimator is not fit yet. Please run fit() before predict().")+ return None++ @classmethod+ def _search_space(cls, **params):+ # TODO: fill in a proper search space+ space = {}+ return space+++class Prophet(TimeSeriesEstimator):+ """The class for tuning Prophet."""++ @classmethod+ def _search_space(cls, **params):+ space = {+ "changepoint_prior_scale": {+ "domain": tune.loguniform(lower=0.001, upper=0.05),+ "init_value": 0.05,+ "low_cost_init_value": 0.001,+ },+ "seasonality_prior_scale": {+ "domain": tune.loguniform(lower=0.01, upper=10),+ "init_value": 10,+ },+ "holidays_prior_scale": {+ "domain": tune.loguniform(lower=0.01, upper=10),+ "init_value": 10,+ },+ "seasonality_mode": {+ "domain": tune.choice(["additive", "multiplicative"]),+ "init_value": "multiplicative",+ },+ }+ return space++ def fit(self, X_train, y_train=None, budget=None, **kwargs):+ from prophet import Prophet++ X_train = self.enrich(X_train)+ super().fit(X_train, y_train, budget=budget, **kwargs)++ current_time = time.time()++ if isinstance(X_train, TimeSeriesDataset):+ data = X_train+ target_col = data.target_names[0]+ time_col = data.time_col+ regressors = data.regressors+ # this class only supports univariate regression+ train_df = data.train_data[regressors + [target_col, time_col]]+ train_df = train_df.rename(columns={target_col: "y", time_col: "ds"})+ else:+ train_df = self._join(X_train, y_train)++ regressors = list(train_df.columns)+ regressors.remove(TS_TIMESTAMP_COL)+ regressors.remove(TS_VALUE_COL)++ train_df = self._preprocess(train_df)+ logging.getLogger("prophet").setLevel(logging.WARNING)+ nice_params = {k: v for k, v in self.params.items() if k in self._search_space()}+ model = Prophet(**nice_params)+ for regressor in regressors:+ model.add_regressor(regressor)+ with suppress_stdout_stderr():+ model.fit(train_df)+ train_time = time.time() - current_time+ self._model = model+ return train_time++ def predict(self, X, **kwargs):+ X = self.enrich(X)+ if isinstance(X, int):+ raise ValueError(+ "predict() with steps is only supported for arima/sarimax."+ " For Prophet, pass a dataframe with the first column containing"+ " the timestamp values."+ )++ if isinstance(X, TimeSeriesDataset):+ data = X+ X = data.test_data[data.regressors + [data.time_col]]++ X = X.rename(columns={self.time_col: "ds"})+ if self._model is not None:+ X = self._preprocess(X)+ forecast = self._model.predict(X, **kwargs)+ out = forecast["yhat"]+ out.name = self.target_names[0]+ return out++ else:+ logger.warning("Estimator is not fit yet. Please run fit() before predict().")+ return np.ones(X.shape[0])+++class StatsModelsEstimator(TimeSeriesEstimator):+ def predict(self, X, **kwargs) -> pd.Series:+ X = self.enrich(X)+ if self._model is None or self._model is False:+ return np.ones(X if isinstance(X, int) else X.shape[0])++ if isinstance(X, int):+ return self._model.forecast(steps=X)++ if isinstance(X, TimeSeriesDataset):+ data = X+ X = data.test_data[data.regressors + [data.time_col]]+ else:+ X = X[self.regressors + [self.time_col]]++ if isinstance(X, DataFrame):+ start = X[self.time_col].iloc[0]+ end = X[self.time_col].iloc[-1]+ if len(self.regressors):+ exog = self._preprocess(X[self.regressors])+ forecast = self._model.predict(start=start, end=end, exog=exog.values, **kwargs)+ else:+ forecast = self._model.predict(start=start, end=end, **kwargs)+ else:+ raise ValueError(+ "X needs to be either a pandas Dataframe with dates as the first column"+ " or an int number of periods for predict()."+ )+ forecast.name = self.target_names[0]+ return forecast+++class ARIMA(StatsModelsEstimator):+ """The class for tuning ARIMA."""++ def __init__(self, **kwargs):+ super().__init__(**kwargs)+ if not all([p in self.params for p in ["p", "d", "q"]]):+ print("arima params at init time:")+ print(self.params)+ try:+ raise ValueError("ARIMA initialized without required params p, d, q")+ except Exception as e:+ import traceback++ print(traceback.format_exc())+ raise e++ @classmethod+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):+ scale, _ = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)+ space = {+ "p": {+ "domain": tune.qrandint(lower=0, upper=2 * scale, q=1),+ "init_value": scale,+ "low_cost_init_value": 0,+ },+ "d": {+ "domain": tune.qrandint(lower=0, upper=6, q=1),+ "init_value": 1,+ "low_cost_init_value": 0,+ },+ "q": {+ "domain": tune.qrandint(lower=0, upper=2 * scale, q=1),+ "init_value": scale,+ "low_cost_init_value": 0,+ },+ }+ return space++ def _join(self, X_train, y_train):+ train_df = super()._join(X_train, y_train)+ train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])+ train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)+ return train_df++ def fit(self, X_train, y_train=None, budget=None, **kwargs):+ import warnings++ super().fit(X_train, y_train, budget=budget, **kwargs)+ X_train = self.enrich(X_train, remove_constants=True)++ warnings.filterwarnings("ignore")+ from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator++ current_time = time.time()++ if isinstance(X_train, TimeSeriesDataset):+ data = X_train+ # this class only supports univariate regression+ target_col = data.target_names[0] if isinstance(data.target_names, list) else data.target_names+ self.regressors = data.regressors+ train_df = data.train_data[self.regressors + [target_col]]+ train_df.index = to_datetime(data.train_data[data.time_col])+ self.time_col = data.time_col+ self.target_names = target_col+ else:+ target_col = TS_VALUE_COL+ train_df = self._join(X_train, y_train)+ self.regressors = list(train_df)+ self.regressors.remove(TS_VALUE_COL)++ train_df = self._preprocess(train_df)++ if len(self.regressors):+ model = ARIMA_estimator(+ train_df[[target_col]],+ exog=train_df[self.regressors],+ order=(self.params["p"], self.params["d"], self.params["q"]),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ else:+ model = ARIMA_estimator(+ train_df,+ order=(self.params["p"], self.params["d"], self.params["q"]),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ with suppress_stdout_stderr():+ model = model.fit()+ train_time = time.time() - current_time+ self._model = model+ return train_time+++class SARIMAX(StatsModelsEstimator):+ """The class for tuning SARIMA."""++ @classmethod+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):+ scale, max_lags = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)++ # TODO: instead, downscale the dataset and take next_scale from that for P and Q+ scales = [+ s for s in [scale, 2 * scale, 3 * scale, 4 * scale] if s * max_lags <= len(data.train_data) - pred_horizon+ ]++ space = {+ "p": {+ "domain": tune.qrandint(lower=0, upper=scale - 1, q=1),+ "init_value": scale - 1,+ "low_cost_init_value": 0,+ },+ "d": {+ "domain": tune.qrandint(lower=0, upper=6, q=1),+ "init_value": 0,+ "low_cost_init_value": 0,+ },+ "q": {+ "domain": tune.qrandint(lower=0, upper=scale - 1, q=1),+ "init_value": scale - 1,+ "low_cost_init_value": 0,+ },+ "P": {+ "domain": tune.qrandint(lower=0, upper=min(10, max_lags), q=1),+ "init_value": 3,+ "low_cost_init_value": 0,+ },+ "D": {+ "domain": tune.qrandint(lower=0, upper=6, q=1),+ "init_value": 0,+ "low_cost_init_value": 0,+ },+ "Q": {+ "domain": tune.qrandint(lower=0, upper=min(10, max_lags), q=1),+ "init_value": 3,+ "low_cost_init_value": 0,+ },+ "s": {+ "domain": tune.choice(scales),+ "init_value": scale,+ },+ }+ return space++ def fit(self, X_train, y_train=None, budget=None, **kwargs):+ import warnings++ super().fit(X_train, y_train, budget=budget, **kwargs)+ X_train = self.enrich(X_train)++ warnings.filterwarnings("ignore")+ from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator++ current_time = time.time()++ if isinstance(X_train, TimeSeriesDataset):+ data = X_train+ target_col = data.target_names[0]+ self.regressors = data.regressors+ # this class only supports univariate regression+ train_df = data.train_data[self.regressors + [target_col]]+ train_df.index = to_datetime(data.train_data[data.time_col])+ else:+ target_col = TS_VALUE_COL+ train_df = self._join(X_train, y_train)+ self.regressors = list(train_df)+ self.regressors.remove(TS_VALUE_COL)++ train_df = self._preprocess(train_df)+ # regressors = list(train_df)+ # regressors.remove(target_col)+ if self.regressors:+ model = SARIMAX_estimator(+ train_df[[target_col]],+ exog=train_df[self.regressors],+ order=(self.params["p"], self.params["d"], self.params["q"]),+ seasonal_order=(+ self.params["P"],+ self.params["D"],+ self.params["Q"],+ self.params["s"],+ ),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ else:+ model = SARIMAX_estimator(+ train_df,+ order=(self.params["p"], self.params["d"], self.params["q"]),+ seasonal_order=(+ self.params["P"],+ self.params["D"],+ self.params["Q"],+ self.params["s"],+ ),+ enforce_stationarity=False,+ enforce_invertibility=False,+ )+ with suppress_stdout_stderr():+ model = model.fit()+ train_time = time.time() - current_time+ self._model = model+ return train_time+++class HoltWinters(StatsModelsEstimator):+ """+ The class for tuning Holt Winters model, aka 'Triple Exponential Smoothing'.+ """++ @classmethod+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):+ space = {+ "damped_trend": {"domain": tune.choice([True, False]), "init_value": False},+ "trend": {"domain": tune.choice(["add", "mul", None]), "init_value": "add"},+ "seasonal": {+ "domain": tune.choice(["add", "mul", None]),+ "init_value": "add",+ },+ "use_boxcox": {"domain": tune.choice([False, True]), "init_value": False},+ "seasonal_periods": { # statsmodels casts this to None if "seasonal" is None+ "domain": tune.choice([7, 12, 4, 52, 6]), # weekly, yearly, quarterly, weekly w yearly data+ "init_value": 7,+ },+ }+ return space++ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):+ import warnings++ warnings.filterwarnings("ignore")+ from statsmodels.tsa.holtwinters import (+ ExponentialSmoothing as HWExponentialSmoothing,+ )++ current_time = time.time()+ super().fit(X_train, y_train, budget=budget, **kwargs)+ X_train = self.enrich(X_train)++ self.regressors = []+ if isinstance(X_train, TimeSeriesDataset):+ data = X_train+ target_col = data.target_names[0]+ regressors = data.regressors+ # this class only supports univariate regression+ train_df = data.train_data[self.regressors + [target_col]]+ train_df.index = to_datetime(data.train_data[data.time_col])+ else:+ target_col = TS_VALUE_COL+ train_df = self._join(X_train, y_train)+ regressors = list(train_df)+ regressors.remove(TS_VALUE_COL)++ if regressors:+ logger.warning("Regressors are ignored for Holt-Winters ETS models.")++ train_df = self._preprocess(train_df)++ # Override incompatible parameters+ if (+ train_df.shape[0] < 2 * self.params["seasonal_periods"]+ ): # this would prevent heuristic initialization to work properly+ self.params["seasonal"] = None+ if (+ self.params["seasonal"] == "mul" and (train_df.y == 0).sum() > 0+ ): # cannot have multiplicative seasonality in this case+ self.params["seasonal"] = "add"+ if self.params["trend"] == "mul" and (train_df.y == 0).sum() > 0:+ self.params["trend"] = "add"++ if not self.params["seasonal"] or self.params["trend"] not in ["mul", "add"]:+ self.params["damped_trend"] = False++ model = HWExponentialSmoothing(+ train_df[[target_col]],+ damped_trend=self.params["damped_trend"],+ seasonal=self.params["seasonal"],+ trend=self.params["trend"],+ )+ with suppress_stdout_stderr():+ model = model.fit()+ train_time = time.time() - current_time+ self._model = model+ return train_time+++class TS_SKLearn(TimeSeriesEstimator):+ """The class for tuning SKLearn Regressors for time-series forecasting"""++ base_class = SKLearnEstimator++ @classmethod+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):+ data_size = data.train_data.shape+ space = cls.base_class.search_space(data_size=data_size, task=task, **params)++ scale, _ = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)++ max_lags = max(3 * scale, int(np.sqrt(data_size[0])))+ max_lags = min(max_lags, data_size[0] - pred_horizon - 1)++ space.update(+ {+ "lags": {+ "domain": tune.randint(lower=1, upper=max_lags),+ "init_value": min(max_lags, scale),+ },+ }+ )+ return space++ def __init__(self, task="ts_forecast", **params):+ # TODO: pass task objects throughout+ super().__init__(task, **params)+ self._model = None+ self.ts_task = task++ def fit(self, X_train, y_train=None, budget=None, **kwargs):+ super().fit(X_train, y_train, budget=budget, **kwargs)+ X_train = self.enrich(X_train)++ current_time = time.time()+ if isinstance(X_train, TimeSeriesDataset):+ data = X_train+ X_train = data.train_data[data.regressors + [data.time_col]]+ self.regressors = data.regressors+ # this class only supports univariate regression+ y_train = data.y_train+ self.time_col = data.time_col+ self.target_names = data.target_names+ elif isinstance(X_train, DataFrame):+ self.time_col = X_train.columns.tolist()[0]++ # X_train = self.transform_X(X_train)+ self.regressors = X_train.columns.tolist()[1:]+ else:+ raise ValueError("Unknown X type")++ X_train = self._preprocess(X_train)++ est_params = {k: v for k, v in self.params.items() if k not in self.top_search_space().keys()}++ from flaml.automl.time_series.sklearn import SklearnWrapper++ horizon = kwargs.pop("period")+ lags = est_params.pop("lags")+ est_params["task"] = self._task+ self._model = SklearnWrapper(+ self.base_class,+ horizon=horizon,+ lags=lags,+ init_params=est_params,+ pca_features=self.params.get("pca_features", False),+ )+ self._model.fit(X_train[self.regressors], y_train)++ train_time = time.time() - current_time+ return train_time++ def predict(self, X, **kwargs):+ X = self.enrich(X)+ if isinstance(X, TimeSeriesDataset):+ data = X+ X = data.test_data++ if self._model is not None:+ X = X[self.regressors]+ # X = self.transform_X(X)+ X = self._preprocess(X)+ forecast = self._model.predict(X)+ if isinstance(forecast, Series):+ forecast.name = self.target_names[0]++ return forecast+ else:+ logger.warning("Estimator is not fit yet. Please run fit() before predict().")+ return np.ones(X.shape[0])+++class LGBM_TS(TS_SKLearn):+ """The class for tuning LGBM Regressor for time-series forecasting"""++ base_class = LGBMEstimator+++class XGBoost_TS(TS_SKLearn):+ """The class for tuning XGBoost Regressor for time-series forecasting"""++ base_class = XGBoostSklearnEstimator+++class RF_TS(TS_SKLearn):+ """The class for tuning Random Forest Regressor for time-series forecasting"""++ base_class = RandomForestEstimator+++class ExtraTrees_TS(TS_SKLearn):+ """The class for tuning Extra Trees Regressor for time-series forecasting"""++ base_class = ExtraTreesEstimator+++class XGBoostLimitDepth_TS(TS_SKLearn):+ """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""++ base_class = XGBoostLimitDepthEstimator+++# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball+class CatBoost_TS(TS_SKLearn):+ base_class = CatBoostEstimatordiff --git a/flaml/automl/training_log.py b/flaml/automl/training_log.pyindex c49c6e592..0c01c3f6a 100644--- a/flaml/automl/training_log.py+++ b/flaml/automl/training_log.py@@ -29,7 +29,7 @@ class TrainingLogRecord(object):self.logged_metric = logged_metricself.trial_time = trial_timeself.wall_clock_time = wall_clock_time- self.validation_loss = validation_loss+ self.validation_loss = float(validation_loss)self.config = configself.learner = learnerself.sample_size = sample_sizediff --git a/flaml/default/suggest.py b/flaml/default/suggest.pyindex 12acc3678..05ff342eb 100644--- a/flaml/default/suggest.py+++ b/flaml/default/suggest.py@@ -5,7 +5,7 @@ import jsonfrom flaml.automl.data import DataTransformerfrom flaml.automl.task.task import CLASSIFICATION, get_classification_objectivefrom flaml.automl.task.generic_task import len_labels-from flaml.automl.ml import get_estimator_class+from flaml.automl.task.factory import task_factoryfrom flaml.version import __version__try:@@ -162,9 +162,10 @@ def suggest_hyperparams(task, X, y, estimator_or_predictor, location=None):"""config = suggest_config(task, X, y, estimator_or_predictor, location=location, k=1)[0]estimator = config["class"]- model_class = get_estimator_class(task, estimator)+ task = task_factory(task)+ model_class = task.estimator_class_from_str(estimator)hyperparams = config["hyperparameters"]- model = model_class(task=task, **hyperparams)+ model = model_class(task=task.name, **hyperparams)estimator_class = model.estimator_classhyperparams = hyperparams and model.paramsreturn hyperparams, estimator_class@@ -246,7 +247,7 @@ def preprocess_and_suggest_hyperparams()config = suggest_config(task, X, y, estimator_or_predictor, location=location, k=1)[0]estimator = config["class"]- model_class = get_estimator_class(task, estimator)+ model_class = task_factory(task).estimator_class_from_str(estimator)hyperparams = config["hyperparameters"]model = model_class(task=task, **hyperparams)if model.estimator_class is None:diff --git a/setup.py b/setup.pyindex c29272b38..fd588084d 100644--- a/setup.py+++ b/setup.py@@ -81,6 +81,7 @@ setuptools.setup("nbformat","ipykernel","pytorch-lightning<1.9.1", # test_forecast_panel+ "tensorboardX==2.6", # test_forecast_panel"requests<2.29.0", # https://github.com/docker/docker-py/issues/3113"packaging",],@@ -125,6 +126,8 @@ setuptools.setup("statsmodels>=0.12.2","hcrystalball==0.1.10","pytorch-forecasting>=0.9.0",+ "pytorch-lightning==1.9.0",+ "tensorboardX==2.6",],"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],"openai": ["openai==0.27.8", "diskcache"],diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.pyindex 044d11a72..19997c3c8 100644--- a/test/automl/test_forecast.py+++ b/test/automl/test_forecast.py@@ -1,8 +1,14 @@+import datetime+import numpy as np+import pandas as pd+from flaml import AutoML+from flaml.automl.task.time_series_task import TimeSeriesTask-def test_forecast_automl(budget=5, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]):++def test_forecast_automl(budget=10, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]):# using dataframeimport statsmodels.api as sm@@ -88,6 +94,32 @@ def test_forecast_automl(budget=5, estimators_when_no_prophet=["arima", "sarimax)+def test_models(budget=3):+ n = 100+ X = pd.DataFrame(+ {+ "A": pd.date_range(start="1900-01-01", periods=n, freq="D"),+ }+ )+ y = np.exp(np.random.randn(n))++ task = TimeSeriesTask("ts_forecast")++ for est in task.estimators.keys():+ if est == "tft":+ continue # TFT is covered by its own test+ automl = AutoML()+ automl.fit(+ X_train=X[:72], # a single column of timestamp+ y_train=y[:72], # value for each timestamp+ estimator_list=[est],+ period=12, # time horizon to forecast, e.g., 12 months+ task="ts_forecast",+ time_budget=budget, # time budget in seconds+ )+ automl.predict(X[72:])++def test_numpy():X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]")y_train = np.random.random(size=len(X_train))@@ -500,7 +532,7 @@ def test_forecast_panel(budget=5):],"time_varying_unknown_categoricals": [],"time_varying_unknown_reals": [- "y", # always need a 'y' column for the target column+ "volume", # target column"log_volume","industry_volume","soda_volume",@@ -572,10 +604,69 @@ def test_forecast_panel(budget=5):print(automl.min_resource)+def test_cv_step():+ n = 300+ time_col = "date"+ df = pd.DataFrame(+ {+ time_col: pd.date_range(start="1/1/2001", periods=n, freq="D"),+ "y": np.sin(np.linspace(start=0, stop=200, num=n)),+ }+ )++ def split_by_date(df: pd.DataFrame, dt: datetime.date):+ dt = datetime.datetime(dt.year, dt.month, dt.day)+ return df[df[time_col] <= dt], df[df[time_col] > dt]++ horizon = 60+ data_end = df.date.max()+ train_end = data_end - datetime.timedelta(days=horizon)++ train_df, val_df = split_by_date(df, train_end)+ from flaml import AutoML++ tgts = ["y"]+ # tgt = "SERIES_SANCTIONS"++ preds = {}+ for tgt in tgts:+ features = [] # [c for c in train_df.columns if "SERIES" not in c and c != time_col]++ automl = AutoML(time_budget=5, metric="mae", task="ts_forecast", eval_method="cv")++ automl.fit(+ dataframe=train_df[[time_col] + features + [tgt]],+ label=tgt,+ period=horizon,+ time_col=time_col,+ verbose=4,+ n_splits=5,+ cv_step_size=5,+ )++ pred = automl.predict(val_df)++ if isinstance(pred, pd.DataFrame):+ pred = pred[tgt]+ assert not np.isnan(pred.sum())++ import matplotlib.pyplot as plt++ preds[tgt] = pred+ # plt.figure(figsize=(16, 8), dpi=80)+ # plt.plot(df[time_col], df[tgt])+ # plt.plot(val_df[time_col], pred)+ # plt.legend(["actual", "predicted"])+ # plt.show()++ print("yahoo!")++if __name__ == "__main__":- test_forecast_automl(60)- test_multivariate_forecast_num(5)- test_multivariate_forecast_cat(5)- test_numpy()- test_forecast_classification(5)+ # test_forecast_automl(60)+ # test_multivariate_forecast_num(5)+ # test_multivariate_forecast_cat(5)+ # test_numpy()+ # test_forecast_classification(5)test_forecast_panel(5)+ # test_cv_step()diff --git a/test/test_model.py b/test/test_model.pyindex b05f5d58a..ab4d89397 100644--- a/test/test_model.py+++ b/test/test_model.py@@ -10,10 +10,8 @@ from flaml.automl.model import (CatBoostEstimator,XGBoostEstimator,RandomForestEstimator,- Prophet,- ARIMA,- LGBM_TS,)+from flaml.automl.time_series import Prophet, ARIMA, LGBM_TS, TimeSeriesDatasetdef test_lrl2():@@ -101,7 +99,8 @@ def test_prep():passprophet.predict(X)- arima = ARIMA()+ # What's the point of callin ARIMA without parameters, or calling predict before fit?+ arima = ARIMA(p=1, q=1, d=0)arima.predict(X)arima._model = Falsetry:@@ -109,23 +108,27 @@ def test_prep():except ValueError:# X_test needs to be either a pandas Dataframe with dates as the first column or an int number of periods for predict().pass-- lgbm = LGBM_TS(optimize_for_horizon=True, lags=1)+ lgbm = LGBM_TS(lags=1)X = DataFrame({"A": [- datetime(1900, 2, 3),+ datetime(1900, 3, 1),+ datetime(1900, 3, 2),+ datetime(1900, 3, 3),datetime(1900, 3, 4),datetime(1900, 3, 4),datetime(1900, 3, 4),- datetime(1900, 7, 2),- datetime(1900, 8, 9),+ datetime(1900, 3, 5),+ datetime(1900, 3, 6),],})- y = np.array([0, 1, 0, 1, 0, 0])+ y = np.array([0, 1, 0, 1, 1, 1, 0, 0])lgbm.predict(X[:2])- lgbm.fit(X, y, period=2)+ df = X.copy()+ df["y"] = y+ tsds = TimeSeriesDataset(df, time_col="A", target_names="y")+ lgbm.fit(tsds, period=2)lgbm.predict(X[:2])print(lgbm.feature_names_in_)print(lgbm.feature_importances_)