Skip to content

A programming framework for agentic AI

Factor out time series-related functionality into a time series Task object (#989)

EgorKraevTransferwise5245efbd2c21
diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py
index cdf17a37c..c53f70ec8 100644
--- a/flaml/automl/automl.py
+++ b/flaml/automl/automl.py
@@ -13,10 +13,9 @@ import logging
import json
from flaml.automl.state import SearchState, AutoMLState
-from flaml.automl.ml import (
- train_estimator,
- get_estimator_class,
-)
+from flaml.automl.ml import train_estimator
+
+from flaml.automl.time_series import TimeSeriesDataset
from flaml.config import (
MIN_SAMPLE_TRAIN,
MEM_THRES,
@@ -29,7 +28,7 @@ from flaml.config import (
)
# TODO check to see when we can remove these
-from flaml.automl.task.task import CLASSIFICATION, TS_FORECAST, Task
+from flaml.automl.task.task import CLASSIFICATION, Task
from flaml.automl.task.factory import task_factory
from flaml import tune
from flaml.automl.logger import logger, logger_formatter
@@ -899,7 +898,9 @@ class AutoML(BaseEstimator):
], "eval_method must be 'auto' or 'cv' for custom data splitter."
assert self._state.X_val is None, "custom splitter and custom validation data can't be used together."
return "cv"
- if self._state.X_val is not None:
+ if self._state.X_val is not None and (
+ not isinstance(self._state.X_val, TimeSeriesDataset) or len(self._state.X_val.test_data) > 0
+ ):
assert eval_method in [
"auto",
"holdout",
@@ -1144,7 +1145,7 @@ class AutoML(BaseEstimator):
self._df,
self._sample_weight_full,
)
- self.data_size_full = len(self._state.y_train_all)
+ self.data_size_full = self._state.data_size_full
def fit(
self,
@@ -1196,6 +1197,7 @@ class AutoML(BaseEstimator):
free_mem_ratio=0,
metric_constraints=None,
custom_hp=None,
+ time_col=None,
cv_score_agg_func=None,
skip_transform=None,
mlflow_logging=None,
@@ -1428,6 +1430,8 @@ class AutoML(BaseEstimator):
}
}
```
+ time_col: for a time series task, name of the column containing the timestamps. If not
+ provided, defaults to the first column of X_train/X_val
cv_score_agg_func: customized cross-validation scores aggregate function. Default to average metrics across folds. If specificed, this function needs to
have the following input arguments:
@@ -1522,6 +1526,7 @@ class AutoML(BaseEstimator):
if isinstance(task, str):
task = task_factory(task, X_train, y_train)
self._state.task = task
+ self._state.task.time_col = time_col
self._estimator_type = "classifier" if task.is_classification() else "regressor"
time_budget = time_budget or self._settings.get("time_budget")
n_jobs = n_jobs or self._settings.get("n_jobs")
@@ -1824,7 +1829,7 @@ class AutoML(BaseEstimator):
if estimator_name not in self._state.learner_classes:
self.add_learner(
estimator_name,
- get_estimator_class(self._state.task, estimator_name),
+ self._state.task.estimator_class_from_str(estimator_name),
)
# set up learner search space
if isinstance(starting_points, str) and starting_points.startswith("data"):
@@ -1879,7 +1884,8 @@ class AutoML(BaseEstimator):
self._search_states[estimator_name] = SearchState(
learner_class=estimator_class,
- data_size=self._state.data_size,
+ # data_size=self._state.data_size,
+ data=self._state.X_train,
task=self._state.task,
starting_point=starting_points.get(estimator_name),
period=self._state.fit_kwargs.get(
@@ -2589,7 +2595,7 @@ class AutoML(BaseEstimator):
if self._max_iter > 1:
self._state.time_budget = -1
if (
- self._state.task in TS_FORECAST
+ self._state.task.is_ts_forecast()
or self._trained_estimator is None
or self._trained_estimator.model is None
or (
diff --git a/flaml/automl/ml.py b/flaml/automl/ml.py
index 7040088f3..c14ba5cdd 100644
--- a/flaml/automl/ml.py
+++ b/flaml/automl/ml.py
@@ -3,36 +3,16 @@
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
import time
-import numpy as np
from typing import Union, Callable, TypeVar, Optional, Tuple
-from flaml.automl.model import (
- XGBoostSklearnEstimator,
- XGBoost_TS,
- XGBoostLimitDepthEstimator,
- XGBoostLimitDepth_TS,
- RandomForestEstimator,
- RF_TS,
- LGBMEstimator,
- LGBM_TS,
- LRL1Classifier,
- LRL2Classifier,
- CatBoostEstimator,
- ExtraTreesEstimator,
- ExtraTrees_TS,
- KNeighborsEstimator,
- Prophet,
- ARIMA,
- SARIMAX,
- HoltWinters,
- TransformersEstimator,
- TemporalFusionTransformerEstimator,
- TransformersEstimatorModelSelection,
- SparkLGBMEstimator,
-)
+import logging
+
+import numpy as np
+
+
from flaml.automl.data import group_counts
-from flaml.automl.task.task import TS_FORECAST, Task
-from flaml.automl.model import BaseEstimator
-from flaml.automl.spark import psDataFrame, psSeries, ERROR as SPARK_ERROR, Series
+from flaml.automl.task.task import Task
+from flaml.automl.model import BaseEstimator, TransformersEstimator
+from flaml.automl.spark import psDataFrame, psSeries, ERROR as SPARK_ERROR, Series, DataFrame
try:
from sklearn.metrics import (
@@ -53,6 +33,11 @@ except ImportError:
if SPARK_ERROR is None:
from flaml.automl.spark.metrics import spark_metric_loss_score
+from flaml.automl.time_series import TimeSeriesDataset
+
+logger = logging.getLogger(__name__)
+
+
EstimatorSubclass = TypeVar("EstimatorSubclass", bound=BaseEstimator)
sklearn_metric_name_set = {
@@ -106,55 +91,6 @@ huggingface_metric_to_mode = {
huggingface_submetric_to_metric = {"rouge1": "rouge", "rouge2": "rouge"}
-def get_estimator_class(task: str, estimator_name: str) -> EstimatorSubclass:
- """Given a task and an estimator name, return the relevant flaml-wrapped estimator class
-
- NOTE: See why the return type is declarad by using TypeVar here on the mypy doc
- https://mypy.readthedocs.io/en/stable/kinds_of_types.html#the-type-of-class-objects
- """
- # when adding a new learner, need to add an elif branch
- if "xgboost" == estimator_name:
- estimator_class = XGBoost_TS if task in TS_FORECAST else XGBoostSklearnEstimator
- elif "xgb_limitdepth" == estimator_name:
- estimator_class = XGBoostLimitDepth_TS if task in TS_FORECAST else XGBoostLimitDepthEstimator
- elif "rf" == estimator_name:
- estimator_class = RF_TS if task in TS_FORECAST else RandomForestEstimator
- elif "lgbm" == estimator_name:
- estimator_class = LGBM_TS if task in TS_FORECAST else LGBMEstimator
- elif "lgbm_spark" == estimator_name:
- estimator_class = SparkLGBMEstimator
- elif "lrl1" == estimator_name:
- estimator_class = LRL1Classifier
- elif "lrl2" == estimator_name:
- estimator_class = LRL2Classifier
- elif "catboost" == estimator_name:
- estimator_class = CatBoostEstimator
- elif "extra_tree" == estimator_name:
- estimator_class = ExtraTrees_TS if task in TS_FORECAST else ExtraTreesEstimator
- elif "kneighbor" == estimator_name:
- estimator_class = KNeighborsEstimator
- elif "prophet" in estimator_name:
- estimator_class = Prophet
- elif estimator_name == "arima":
- estimator_class = ARIMA
- elif estimator_name == "sarimax":
- estimator_class = SARIMAX
- elif estimator_name == "holt-winters":
- estimator_class = HoltWinters
- elif estimator_name == "transformer":
- estimator_class = TransformersEstimator
- elif estimator_name == "tft":
- estimator_class = TemporalFusionTransformerEstimator
- elif estimator_name == "transformer_ms":
- estimator_class = TransformersEstimatorModelSelection
- else:
- raise ValueError(
- estimator_name + " is not a built-in learner. "
- "Please use AutoML.add_learner() to add a customized learner."
- )
- return estimator_class
-
-
def metric_loss_score(
metric_name: str,
y_processed_predict,
@@ -267,6 +203,7 @@ def sklearn_metric_loss_score(
Returns:
score: A float number of the loss, the lower the better.
"""
+
metric_name = metric_name.lower()
if "r2" == metric_name:
@@ -356,132 +293,20 @@ def get_y_pred(estimator, X, eval_metric, task: Task):
y_pred = estimator.predict_proba(X)
else:
y_pred = estimator.predict(X)
- return y_pred
+ if isinstance(y_pred, Series) or isinstance(y_pred, DataFrame):
+ y_pred = y_pred.values
-def _eval_estimator(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric: Union[str, Callable],
- task,
- labels=None,
- log_training_metric=False,
- fit_kwargs: Optional[dict] = None,
-):
- if fit_kwargs is None:
- fit_kwargs = {}
- if isinstance(eval_metric, str):
- pred_start = time.time()
- val_pred_y = get_y_pred(estimator, X_val, eval_metric, task)
- pred_time = (time.time() - pred_start) / X_val.shape[0]
+ return y_pred
- val_loss = metric_loss_score(
- eval_metric,
- y_processed_predict=val_pred_y,
- y_processed_true=y_val,
- labels=labels,
- sample_weight=weight_val,
- groups=groups_val,
- )
- metric_for_logging = {"pred_time": pred_time}
- if log_training_metric:
- train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)
- metric_for_logging["train_loss"] = metric_loss_score(
- eval_metric,
- train_pred_y,
- y_train,
- labels,
- fit_kwargs.get("sample_weight"),
- fit_kwargs.get("groups"),
- )
- else: # customized metric function
- val_loss, metric_for_logging = eval_metric(
- X_val,
- y_val,
- estimator,
- labels,
- X_train,
- y_train,
- weight_val,
- fit_kwargs.get("sample_weight"),
- config,
- groups_val,
- fit_kwargs.get("groups"),
- )
- pred_time = metric_for_logging.get("pred_time", 0)
- val_pred_y = None
- # eval_metric may return val_pred_y but not necessarily. Setting None for now.
- return val_loss, metric_for_logging, pred_time, val_pred_y
+def to_numpy(x):
+ if isinstance(x, Series or isinstance(x, DataFrame)):
+ x = x.values
+ else:
+ x = np.ndarray(x)
-def get_val_loss(
- config,
- estimator: EstimatorSubclass,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric: Union[str, Callable],
- obj,
- labels=None,
- budget=None,
- log_training_metric=False,
- fit_kwargs: Optional[dict] = None,
- free_mem_ratio=0,
-):
- if fit_kwargs is None:
- fit_kwargs = {}
- start = time.time()
- # if groups_val is not None:
- # fit_kwargs['groups_val'] = groups_val
- # fit_kwargs['X_val'] = X_val
- # fit_kwargs['y_val'] = y_val
- estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)
- val_loss, metric_for_logging, pred_time, _ = _eval_estimator(
- config,
- estimator,
- X_train,
- y_train,
- X_val,
- y_val,
- weight_val,
- groups_val,
- eval_metric,
- obj,
- labels,
- log_training_metric,
- fit_kwargs,
- )
- if hasattr(estimator, "intermediate_results"):
- metric_for_logging["intermediate_results"] = estimator.intermediate_results
- train_time = time.time() - start
- return val_loss, metric_for_logging, train_time, pred_time
-
-
-def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):
- metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)
- metrics_to_log = None
- for single_fold in log_metrics_folds:
- if metrics_to_log is None:
- metrics_to_log = single_fold
- elif isinstance(metrics_to_log, dict):
- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
- else:
- metrics_to_log += single_fold
- if metrics_to_log:
- n = len(val_loss_folds)
- metrics_to_log = (
- {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n
- )
- return metric_to_minimize, metrics_to_log
+ return x.reshape((-1, 1))
def compute_estimator(
@@ -494,7 +319,7 @@ def compute_estimator(
budget,
kf,
config_dic: dict,
- task: str,
+ task: Union[str, Task],
estimator_name: str,
eval_method: str,
eval_metric: Union[str, Callable],
@@ -509,7 +334,7 @@ def compute_estimator(
if fit_kwargs is None:
fit_kwargs = {}
- estimator_class = estimator_class or get_estimator_class(task, estimator_name)
+ estimator_class = estimator_class or task.estimator_class_from_str(estimator_name)
estimator = estimator_class(
**config_dic,
task=task,
@@ -576,7 +401,7 @@ def train_estimator(
free_mem_ratio=0,
) -> Tuple[EstimatorSubclass, float]:
start_time = time.time()
- estimator_class = estimator_class or get_estimator_class(task, estimator_name)
+ estimator_class = estimator_class or task.estimator_class_from_str(estimator_name)
estimator = estimator_class(
**config_dic,
task=task,
@@ -589,7 +414,7 @@ def train_estimator(
fit_kwargs["metric"] = eval_metric
if X_train is not None:
- train_time = estimator.fit(X_train, y_train, budget, free_mem_ratio, **fit_kwargs)
+ train_time = estimator.fit(X_train, y_train, budget=budget, free_mem_ratio=free_mem_ratio, **fit_kwargs)
else:
estimator = estimator.estimator_class(**estimator.params)
train_time = time.time() - start_time
@@ -642,3 +467,140 @@ def multi_class_curves(
for i in range(len(classes)):
curve_x[i], curve_y[i], _ = curve_func(y_true_binary[:, i], y_pred_proba[:, i])
return curve_x, curve_y
+
+
+def get_val_loss(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ task,
+ labels=None,
+ budget=None,
+ log_training_metric=False,
+ fit_kwargs={},
+ free_mem_ratio=0,
+):
+ start = time.time()
+ # if groups_val is not None:
+ # fit_kwargs['groups_val'] = groups_val
+ # fit_kwargs['X_val'] = X_val
+ # fit_kwargs['y_val'] = y_val
+ estimator.fit(X_train, y_train, budget=budget, free_mem_ratio=free_mem_ratio, **fit_kwargs)
+ val_loss, metric_for_logging, pred_time, _ = _eval_estimator(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ task,
+ labels,
+ log_training_metric,
+ fit_kwargs,
+ )
+ if hasattr(estimator, "intermediate_results"):
+ metric_for_logging["intermediate_results"] = estimator.intermediate_results
+ train_time = time.time() - start
+ return val_loss, metric_for_logging, train_time, pred_time
+
+
+def default_cv_score_agg_func(val_loss_folds, log_metrics_folds):
+ metric_to_minimize = sum(val_loss_folds) / len(val_loss_folds)
+ metrics_to_log = None
+ for single_fold in log_metrics_folds:
+ if metrics_to_log is None:
+ metrics_to_log = single_fold
+ elif isinstance(metrics_to_log, dict):
+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold.items()}
+ else:
+ metrics_to_log += single_fold
+ if metrics_to_log:
+ n = len(val_loss_folds)
+ metrics_to_log = (
+ {k: v / n for k, v in metrics_to_log.items()} if isinstance(metrics_to_log, dict) else metrics_to_log / n
+ )
+ return metric_to_minimize, metrics_to_log
+
+
+def _eval_estimator(
+ config,
+ estimator,
+ X_train,
+ y_train,
+ X_val,
+ y_val,
+ weight_val,
+ groups_val,
+ eval_metric,
+ task,
+ labels=None,
+ log_training_metric=False,
+ fit_kwargs={},
+):
+ if isinstance(eval_metric, str):
+ pred_start = time.time()
+ val_pred_y = get_y_pred(estimator, X_val, eval_metric, task)
+
+ # TODO: why are integer labels being cast to str in the first place?
+
+ if isinstance(val_pred_y, Series) or isinstance(val_pred_y, DataFrame) or isinstance(val_pred_y, np.ndarray):
+ test = val_pred_y if isinstance(val_pred_y, np.ndarray) else val_pred_y.values
+ if not np.issubdtype(test.dtype, np.number):
+ # some NLP models return a list
+ val_pred_y = val_pred_y.astype(str)
+
+ if isinstance(X_val, TimeSeriesDataset):
+ num_val_rows = len(X_val.test_data)
+ y_val = X_val.test_data[X_val.target_names].values.astype(val_pred_y.dtype)
+ y_train = X_val.train_data[X_val.target_names].values.astype(val_pred_y.dtype)
+ else:
+ num_val_rows = X_val.shape[0]
+
+ pred_time = (time.time() - pred_start) / num_val_rows
+
+ val_loss = metric_loss_score(
+ eval_metric,
+ y_processed_predict=val_pred_y,
+ y_processed_true=y_val,
+ labels=labels,
+ sample_weight=weight_val,
+ groups=groups_val,
+ )
+ metric_for_logging = {"pred_time": pred_time}
+ if log_training_metric:
+ train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)
+ metric_for_logging["train_loss"] = metric_loss_score(
+ eval_metric,
+ train_pred_y,
+ y_train,
+ labels,
+ fit_kwargs.get("sample_weight"),
+ fit_kwargs.get("groups"),
+ )
+ else: # customized metric function
+ val_loss, metric_for_logging = eval_metric(
+ X_val,
+ y_val,
+ estimator,
+ labels,
+ X_train,
+ y_train,
+ weight_val,
+ fit_kwargs.get("sample_weight"),
+ config,
+ groups_val,
+ fit_kwargs.get("groups"),
+ )
+ pred_time = metric_for_logging.get("pred_time", 0)
+ val_pred_y = None
+ # eval_metric may return val_pred_y but not necessarily. Setting None for now.
+ return val_loss, metric_for_logging, pred_time, val_pred_y
diff --git a/flaml/automl/model.py b/flaml/automl/model.py
index b1be92f15..fea6e8a6d 100644
--- a/flaml/automl/model.py
+++ b/flaml/automl/model.py
@@ -16,19 +16,16 @@ import math
from flaml import tune
from flaml.automl.data import (
group_counts,
- add_time_idx_col,
- TS_TIMESTAMP_COL,
- TS_VALUE_COL,
)
from flaml.automl.task.task import (
- CLASSIFICATION,
- TS_FORECASTREGRESSION,
+ Task,
SEQCLASSIFICATION,
SEQREGRESSION,
TOKENCLASSIFICATION,
SUMMARIZATION,
NLG_TASKS,
)
+from flaml.automl.task.factory import task_factory
try:
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
@@ -119,13 +116,13 @@ class BaseEstimator:
config: A dictionary containing the hyperparameter names, 'n_jobs' as keys.
n_jobs is the number of parallel threads.
"""
- self._task = task
+ self._task = task if isinstance(task, Task) else task_factory(task, None, None)
self.params = self.config2params(config)
self.estimator_class = self._model = None
if "_estimator_type" in config:
self._estimator_type = self.params.pop("_estimator_type")
else:
- self._estimator_type = "classifier" if task in CLASSIFICATION else "regressor"
+ self._estimator_type = "classifier" if self._task.is_classification() else "regressor"
def get_params(self, deep=False):
params = self.params.copy()
@@ -247,7 +244,7 @@ class BaseEstimator:
train_time = self._fit(X_train, y_train, **kwargs)
except (MemoryError, TimeoutError) as e:
logger.warning(f"{e.__class__} {e}")
- if self._task in CLASSIFICATION:
+ if self._task.is_classification():
model = DummyClassifier()
else:
model = DummyRegressor()
@@ -289,7 +286,7 @@ class BaseEstimator:
Each element at (i,j) is the probability for instance i to be in
class j.
"""
- assert self._task in CLASSIFICATION, "predict_proba() only for classification."
+ assert self._task.is_classification(), "predict_proba() only for classification."
X = self._preprocess(X)
return self._model.predict_proba(X, **kwargs)
@@ -490,7 +487,7 @@ class SparkEstimator(BaseEstimator):
Each element at (i,j) is the probability for instance i to be in
class j.
"""
- assert self._task in CLASSIFICATION, "predict_proba() only for classification."
+ assert self._task.is_classification(), "predict_proba() only for classification."
if self._model is not None:
X = self._preprocess(X, index_col=index_col)
predictions = to_pandas_on_spark(self._model.transform(X), index_col=index_col)
@@ -1111,7 +1108,7 @@ class TransformersEstimator(BaseEstimator):
for key, val in pred_kwargs.items():
setattr(self._training_args, key, val)
- assert self._task in CLASSIFICATION, "predict_proba() only for classification tasks."
+ assert self._task.is_classification(), "predict_proba() only for classification tasks."
X_test, _ = self._tokenize_text(X, **self._kwargs)
test_dataset = Dataset.from_pandas(X_test)
@@ -1297,18 +1294,21 @@ class LGBMEstimator(BaseEstimator):
super().__init__(task, **config)
if "verbose" not in self.params:
self.params["verbose"] = -1
- if "regression" == task:
- from lightgbm import LGBMRegressor
- self.estimator_class = LGBMRegressor
- elif "rank" == task:
+ if self._task.is_classification():
+ from lightgbm import LGBMClassifier
+
+ self.estimator_class = LGBMClassifier
+
+ elif task == "rank":
from lightgbm import LGBMRanker
self.estimator_class = LGBMRanker
else:
- from lightgbm import LGBMClassifier
+ from lightgbm import LGBMRegressor
+
+ self.estimator_class = LGBMRegressor
- self.estimator_class = LGBMClassifier
self._time_per_iter = None
self._train_size = 0
self._mem_per_iter = -1
@@ -1626,11 +1626,13 @@ class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):
self.params["verbosity"] = 0
import xgboost as xgb
- self.estimator_class = xgb.XGBRegressor
if "rank" == task:
self.estimator_class = xgb.XGBRanker
- elif task in CLASSIFICATION:
+ elif self._task.is_classification():
self.estimator_class = xgb.XGBClassifier
+ else:
+ self.estimator_class = xgb.XGBRegressor
+
self._xgb_version = xgb.__version__
def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
@@ -1678,7 +1680,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
def search_space(cls, data_size, task, **params):
RandomForestEstimator.nrows = int(data_size[0])
upper = min(2048, RandomForestEstimator.nrows)
- init = 1 / np.sqrt(data_size[1]) if task in CLASSIFICATION else 1
+ init = 1 / np.sqrt(data_size[1]) if task.is_classification() else 1
lower = min(0.1, init)
space = {
"n_estimators": {
@@ -1699,7 +1701,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
"low_cost_init_value": 4,
},
}
- if task in CLASSIFICATION:
+ if task.is_classification():
space["criterion"] = {
"domain": tune.choice(["gini", "entropy"]),
# "init_value": "gini",
@@ -1714,7 +1716,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
params = super().config2params(config)
if "max_leaves" in params:
params["max_leaf_nodes"] = params.get("max_leaf_nodes", params.pop("max_leaves"))
- if self._task not in CLASSIFICATION and "criterion" in config:
+ if not self._task.is_classification() and "criterion" in config:
params.pop("criterion")
if "random_state" not in params:
params["random_state"] = 12032022
@@ -1722,14 +1724,16 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):
def __init__(
self,
- task="binary",
+ task: Task,
**params,
):
super().__init__(task, **params)
self.params["verbose"] = 0
- self.estimator_class = RandomForestRegressor
- if task in CLASSIFICATION:
+
+ if self._task.is_classification():
self.estimator_class = RandomForestClassifier
+ else:
+ self.estimator_class = RandomForestRegressor
class ExtraTreesEstimator(RandomForestEstimator):
@@ -1776,7 +1780,7 @@ class LRL1Classifier(SKLearnEstimator):
def __init__(self, task="binary", **config):
super().__init__(task, **config)
- assert task in CLASSIFICATION, "LogisticRegression for classification task only"
+ assert self._task.is_classification(), "LogisticRegression for classification task only"
self.estimator_class = LogisticRegression
@@ -1802,7 +1806,7 @@ class LRL2Classifier(SKLearnEstimator):
def __init__(self, task="binary", **config):
super().__init__(task, **config)
- assert task in CLASSIFICATION, "LogisticRegression for classification task only"
+ assert self._task.is_classification(), "LogisticRegression for classification task only"
self.estimator_class = LogisticRegression
@@ -1877,13 +1881,14 @@ class CatBoostEstimator(BaseEstimator):
"random_seed": config.get("random_seed", 10242048),
}
)
- from catboost import CatBoostRegressor
-
- self.estimator_class = CatBoostRegressor
- if task in CLASSIFICATION:
+ if self._task.is_classification():
from catboost import CatBoostClassifier
self.estimator_class = CatBoostClassifier
+ else:
+ from catboost import CatBoostRegressor
+
+ self.estimator_class = CatBoostRegressor
def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
start_time = time.time()
@@ -1976,7 +1981,7 @@ class KNeighborsEstimator(BaseEstimator):
def __init__(self, task="binary", **config):
super().__init__(task, **config)
- if task in CLASSIFICATION:
+ if self._task.is_classification():
from sklearn.neighbors import KNeighborsClassifier
self.estimator_class = KNeighborsClassifier
@@ -2003,660 +2008,6 @@ class KNeighborsEstimator(BaseEstimator):
return X
-class Prophet(SKLearnEstimator):
- """The class for tuning Prophet."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "changepoint_prior_scale": {
- "domain": tune.loguniform(lower=0.001, upper=0.05),
- "init_value": 0.05,
- "low_cost_init_value": 0.001,
- },
- "seasonality_prior_scale": {
- "domain": tune.loguniform(lower=0.01, upper=10),
- "init_value": 10,
- },
- "holidays_prior_scale": {
- "domain": tune.loguniform(lower=0.01, upper=10),
- "init_value": 10,
- },
- "seasonality_mode": {
- "domain": tune.choice(["additive", "multiplicative"]),
- "init_value": "multiplicative",
- },
- }
- return space
-
- def __init__(self, task="ts_forecast", n_jobs=1, **params):
- super().__init__(task, **params)
-
- def _join(self, X_train, y_train):
- assert TS_TIMESTAMP_COL in X_train, (
- "Dataframe for training ts_forecast model must have column"
- f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'
- )
- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
- train_df = X_train.join(y_train)
- return train_df
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- from prophet import Prophet
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- cols = list(train_df)
- cols.remove(TS_TIMESTAMP_COL)
- cols.remove(TS_VALUE_COL)
- logging.getLogger("prophet").setLevel(logging.WARNING)
- model = Prophet(**self.params)
- for regressor in cols:
- model.add_regressor(regressor)
- with suppress_stdout_stderr():
- model.fit(train_df)
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def predict(self, X, **kwargs):
- if isinstance(X, int):
- raise ValueError(
- "predict() with steps is only supported for arima/sarimax."
- " For Prophet, pass a dataframe with the first column containing"
- " the timestamp values."
- )
- if self._model is not None:
- X = self._preprocess(X)
- forecast = self._model.predict(X, **kwargs)
- return forecast["yhat"]
- else:
- logger.warning("Estimator is not fit yet. Please run fit() before predict().")
- return np.ones(X.shape[0])
-
- def score(self, X_val: DataFrame, y_val: Series, **kwargs):
- from sklearn.metrics import r2_score
- from .ml import metric_loss_score
-
- y_pred = self.predict(X_val, **kwargs)
- self._metric = kwargs.get("metric", None)
- if self._metric:
- return metric_loss_score(self._metric, y_pred, y_val)
- else:
- return r2_score(y_pred, y_val)
-
-
-class ARIMA(Prophet):
- """The class for tuning ARIMA."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "p": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "d": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- }
- return space
-
- def _join(self, X_train, y_train):
- train_df = super()._join(X_train, y_train)
- train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])
- train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)
- return train_df
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
-
- warnings.filterwarnings("ignore")
- from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- regressors = list(train_df)
- regressors.remove(TS_VALUE_COL)
- if regressors:
- model = ARIMA_estimator(
- train_df[[TS_VALUE_COL]],
- exog=train_df[regressors],
- order=(self.params["p"], self.params["d"], self.params["q"]),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- else:
- model = ARIMA_estimator(
- train_df,
- order=(self.params["p"], self.params["d"], self.params["q"]),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- with suppress_stdout_stderr():
- model = model.fit()
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def predict(self, X, **kwargs):
- if self._model is not None:
- if isinstance(X, int):
- forecast = self._model.forecast(steps=X)
- elif isinstance(X, DataFrame):
- start = X[TS_TIMESTAMP_COL].iloc[0]
- end = X[TS_TIMESTAMP_COL].iloc[-1]
- if len(X.columns) > 1:
- X = self._preprocess(X.drop(columns=TS_TIMESTAMP_COL))
- regressors = list(X)
- forecast = self._model.predict(start=start, end=end, exog=X[regressors], **kwargs)
- else:
- forecast = self._model.predict(start=start, end=end, **kwargs)
- else:
- raise ValueError(
- "X needs to be either a pandas Dataframe with dates as the first column"
- " or an int number of periods for predict()."
- )
- return forecast
- else:
- return np.ones(X if isinstance(X, int) else X.shape[0])
-
-
-class SARIMAX(ARIMA):
- """The class for tuning SARIMA."""
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "p": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "d": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 2,
- "low_cost_init_value": 0,
- },
- "q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "P": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "D": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "Q": {
- "domain": tune.qrandint(lower=0, upper=10, q=1),
- "init_value": 1,
- "low_cost_init_value": 0,
- },
- "s": {
- "domain": tune.choice([1, 4, 6, 12]),
- "init_value": 12,
- },
- }
- return space
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
-
- warnings.filterwarnings("ignore")
- from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- regressors = list(train_df)
- regressors.remove(TS_VALUE_COL)
- if regressors:
- model = SARIMAX_estimator(
- train_df[[TS_VALUE_COL]],
- exog=train_df[regressors],
- order=(self.params["p"], self.params["d"], self.params["q"]),
- seasonal_order=(
- self.params["P"],
- self.params["D"],
- self.params["Q"],
- self.params["s"],
- ),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- else:
- model = SARIMAX_estimator(
- train_df,
- order=(self.params["p"], self.params["d"], self.params["q"]),
- seasonal_order=(
- self.params["P"],
- self.params["D"],
- self.params["Q"],
- self.params["s"],
- ),
- enforce_stationarity=False,
- enforce_invertibility=False,
- )
- with suppress_stdout_stderr():
- model = model.fit()
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
-
-class HoltWinters(ARIMA):
- """
- The class for tuning Holt Winters model, aka 'Triple Exponential Smoothing'.
- """
-
- @classmethod
- def search_space(cls, **params):
- space = {
- "damped_trend": {"domain": tune.choice([True, False]), "init_value": False},
- "trend": {"domain": tune.choice(["add", "mul", None]), "init_value": "add"},
- "seasonal": {
- "domain": tune.choice(["add", "mul", None]),
- "init_value": "add",
- },
- "use_boxcox": {"domain": tune.choice([False, True]), "init_value": False},
- "seasonal_periods": { # statsmodels casts this to None if "seasonal" is None
- "domain": tune.choice([7, 12, 4, 52, 6]), # weekly, yearly, quarterly, weekly w yearly data
- "init_value": 7,
- },
- }
- return space
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
-
- warnings.filterwarnings("ignore")
- from statsmodels.tsa.holtwinters import (
- ExponentialSmoothing as HWExponentialSmoothing,
- )
-
- current_time = time.time()
- train_df = self._join(X_train, y_train)
- train_df = self._preprocess(train_df)
- regressors = list(train_df)
- regressors.remove(TS_VALUE_COL)
- if regressors:
- logger.warning("Regressors are ignored for Holt-Winters ETS models.")
-
- # Override incompatible parameters
- if (
- X_train.shape[0] < 2 * self.params["seasonal_periods"]
- ): # this would prevent heuristic initialization to work properly
- self.params["seasonal"] = None
- if (
- self.params["seasonal"] == "mul" and (train_df.y == 0).sum() > 0
- ): # cannot have multiplicative seasonality in this case
- self.params["seasonal"] = "add"
- if self.params["trend"] == "mul" and (train_df.y == 0).sum() > 0:
- self.params["trend"] = "add"
-
- if not self.params["seasonal"] or self.params["trend"] not in ["mul", "add"]:
- self.params["damped_trend"] = False
-
- model = HWExponentialSmoothing(
- train_df[[TS_VALUE_COL]],
- damped_trend=self.params["damped_trend"],
- seasonal=self.params["seasonal"],
- trend=self.params["trend"],
- )
- with suppress_stdout_stderr():
- model = model.fit()
- train_time = time.time() - current_time
- self._model = model
- return train_time
-
- def predict(self, X, **kwargs):
- if self._model is not None:
- if isinstance(X, int):
- forecast = self._model.forecast(steps=X)
- elif isinstance(X, DataFrame):
- start = X[TS_TIMESTAMP_COL].iloc[0]
- end = X[TS_TIMESTAMP_COL].iloc[-1]
- forecast = self._model.predict(start=start, end=end, **kwargs)
- else:
- raise ValueError(
- "X needs to be either a pandas Dataframe with dates as the first column"
- " or an int number of periods for predict()."
- )
- return forecast
- else:
- return np.ones(X if isinstance(X, int) else X.shape[0])
-
-
-class TS_SKLearn(SKLearnEstimator):
- """The class for tuning SKLearn Regressors for time-series forecasting, using hcrystalball"""
-
- base_class = SKLearnEstimator
-
- @classmethod
- def search_space(cls, data_size, pred_horizon, **params):
- space = cls.base_class.search_space(data_size, **params)
- space.update(
- {
- "optimize_for_horizon": {
- "domain": tune.choice([True, False]),
- "init_value": False,
- "low_cost_init_value": False,
- },
- "lags": {
- "domain": tune.randint(lower=1, upper=max(2, int(np.sqrt(data_size[0])))),
- "init_value": 3,
- },
- }
- )
- return space
-
- def __init__(self, task="ts_forecast", **params):
- super().__init__(task, **params)
- self.hcrystaball_model = None
- self.ts_task = "regression" if task in TS_FORECASTREGRESSION else "classification"
-
- def transform_X(self, X):
- cols = list(X)
- if len(cols) == 1:
- ds_col = cols[0]
- X = DataFrame(index=X[ds_col])
- elif len(cols) > 1:
- ds_col = cols[0]
- exog_cols = cols[1:]
- X = X[exog_cols].set_index(X[ds_col])
- return X
-
- def _fit(self, X_train, y_train, budget=None, **kwargs):
- from hcrystalball.wrappers import get_sklearn_wrapper
-
- X_train = self.transform_X(X_train)
- X_train = self._preprocess(X_train)
- params = self.params.copy()
- lags = params.pop("lags")
- optimize_for_horizon = params.pop("optimize_for_horizon")
- estimator = self.base_class(task=self.ts_task, **params)
- self.hcrystaball_model = get_sklearn_wrapper(estimator.estimator_class)
- self.hcrystaball_model.lags = int(lags)
- self.hcrystaball_model.fit(X_train, y_train)
- if optimize_for_horizon:
- # Direct Multi-step Forecast Strategy - fit a seperate model for each horizon
- model_list = []
- for i in range(1, kwargs["period"] + 1):
- (
- X_fit,
- y_fit,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X_train, y_train, i)
- self.hcrystaball_model.model.set_params(**estimator.params)
- model = self.hcrystaball_model.model.fit(X_fit, y_fit)
- model_list.append(model)
- self._model = model_list
- else:
- (
- X_fit,
- y_fit,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X_train, y_train, kwargs["period"])
- self.hcrystaball_model.model.set_params(**estimator.params)
- model = self.hcrystaball_model.model.fit(X_fit, y_fit)
- self._model = model
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- current_time = time.time()
- self._fit(X_train, y_train, budget=budget, **kwargs)
- train_time = time.time() - current_time
- return train_time
-
- def predict(self, X, **kwargs):
- if self._model is not None:
- X = self.transform_X(X)
- X = self._preprocess(X)
- if isinstance(self._model, list):
- assert len(self._model) == len(
- X
- ), "Model is optimized for horizon, length of X must be equal to `period`."
- preds = []
- for i in range(1, len(self._model) + 1):
- (
- X_pred,
- _,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X.iloc[:i, :])
- preds.append(self._model[i - 1].predict(X_pred, **kwargs)[-1])
- forecast = Series(preds)
- else:
- (
- X_pred,
- _,
- ) = self.hcrystaball_model._transform_data_to_tsmodel_input_format(X)
- forecast = self._model.predict(X_pred, **kwargs)
- return forecast
- else:
- logger.warning("Estimator is not fit yet. Please run fit() before predict().")
- return np.ones(X.shape[0])
-
-
-class LGBM_TS(TS_SKLearn):
- """The class for tuning LGBM Regressor for time-series forecasting"""
-
- base_class = LGBMEstimator
-
-
-class XGBoost_TS(TS_SKLearn):
- """The class for tuning XGBoost Regressor for time-series forecasting"""
-
- base_class = XGBoostSklearnEstimator
-
-
-# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball
-# class CatBoost_TS_Regressor(TS_Regressor):
-# base_class = CatBoostEstimator
-
-
-class RF_TS(TS_SKLearn):
- """The class for tuning Random Forest Regressor for time-series forecasting"""
-
- base_class = RandomForestEstimator
-
-
-class ExtraTrees_TS(TS_SKLearn):
- """The class for tuning Extra Trees Regressor for time-series forecasting"""
-
- base_class = ExtraTreesEstimator
-
-
-class XGBoostLimitDepth_TS(TS_SKLearn):
- """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""
-
- base_class = XGBoostLimitDepthEstimator
-
-
-class TemporalFusionTransformerEstimator(SKLearnEstimator):
- """The class for tuning Temporal Fusion Transformer"""
-
- @classmethod
- def search_space(cls, data_size, pred_horizon, **params):
- space = {
- "gradient_clip_val": {
- "domain": tune.loguniform(lower=0.01, upper=100.0),
- "init_value": 0.01,
- },
- "hidden_size": {
- "domain": tune.lograndint(lower=8, upper=512),
- "init_value": 16,
- },
- "hidden_continuous_size": {
- "domain": tune.randint(lower=1, upper=65),
- "init_value": 8,
- },
- "attention_head_size": {
- "domain": tune.randint(lower=1, upper=5),
- "init_value": 4,
- },
- "dropout": {
- "domain": tune.uniform(lower=0.1, upper=0.3),
- "init_value": 0.1,
- },
- "learning_rate": {
- "domain": tune.loguniform(lower=0.00001, upper=1.0),
- "init_value": 0.001,
- },
- }
- return space
-
- def transform_ds(self, X_train, y_train, **kwargs):
- y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
- self.data = X_train.join(y_train)
-
- max_prediction_length = kwargs["period"]
- self.max_encoder_length = kwargs["max_encoder_length"]
- training_cutoff = self.data["time_idx"].max() - max_prediction_length
-
- from pytorch_forecasting import TimeSeriesDataSet
- from pytorch_forecasting.data import GroupNormalizer
-
- self.group_ids = kwargs["group_ids"].copy()
- training = TimeSeriesDataSet(
- self.data[lambda x: x.time_idx <= training_cutoff],
- time_idx="time_idx",
- target=TS_VALUE_COL,
- group_ids=self.group_ids,
- min_encoder_length=kwargs.get(
- "min_encoder_length", self.max_encoder_length // 2
- ), # keep encoder length long (as it is in the validation set)
- max_encoder_length=self.max_encoder_length,
- min_prediction_length=1,
- max_prediction_length=max_prediction_length,
- static_categoricals=kwargs.get("static_categoricals", []),
- static_reals=kwargs.get("static_reals", []),
- time_varying_known_categoricals=kwargs.get("time_varying_known_categoricals", []),
- time_varying_known_reals=kwargs.get("time_varying_known_reals", []),
- time_varying_unknown_categoricals=kwargs.get("time_varying_unknown_categoricals", []),
- time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),
- variable_groups=kwargs.get(
- "variable_groups", {}
- ), # group of categorical variables can be treated as one variable
- lags=kwargs.get("lags", {}),
- target_normalizer=GroupNormalizer(
- groups=kwargs["group_ids"], transformation="softplus"
- ), # use softplus and normalize by group
- add_relative_time_idx=True,
- add_target_scales=True,
- add_encoder_length=True,
- )
-
- # create validation set (predict=True) which means to predict the last max_prediction_length points in time
- # for each series
- validation = TimeSeriesDataSet.from_dataset(training, self.data, predict=True, stop_randomization=True)
-
- # create dataloaders for model
- batch_size = kwargs.get("batch_size", 64)
- train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
- val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size * 10, num_workers=0)
-
- return training, train_dataloader, val_dataloader
-
- def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
- import warnings
- import pytorch_lightning as pl
- from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
- import torch
- from pytorch_forecasting import TemporalFusionTransformer
- from pytorch_forecasting.metrics import QuantileLoss
-
- warnings.filterwarnings("ignore")
- current_time = time.time()
- training, train_dataloader, val_dataloader = self.transform_ds(X_train, y_train, **kwargs)
- params = self.params.copy()
- gradient_clip_val = params.pop("gradient_clip_val")
- params.pop("n_jobs")
- max_epochs = kwargs.get("max_epochs", 20)
- early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")
-
- def _fit(log):
- default_trainer_kwargs = dict(
- gpus=kwargs.get("gpu_per_trial", [0]) if torch.cuda.is_available() else None,
- max_epochs=max_epochs,
- gradient_clip_val=gradient_clip_val,
- callbacks=[LearningRateMonitor(), early_stop_callback] if log else [early_stop_callback],
- logger=log,
- )
- trainer = pl.Trainer(
- **default_trainer_kwargs,
- )
- tft = TemporalFusionTransformer.from_dataset(
- training,
- **params,
- lstm_layers=2, # 2 is mostly optimal according to documentation
- output_size=7, # 7 quantiles by default
- loss=QuantileLoss(),
- log_interval=10 if log else 0,
- # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches
- reduce_on_plateau_patience=4,
- )
- # fit network
- trainer.fit(
- tft,
- train_dataloaders=train_dataloader,
- val_dataloaders=val_dataloader,
- )
- return trainer
-
- # try:
- # from pytorch_lightning.loggers import TensorBoardLogger
-
- # logger = TensorBoardLogger(
- # kwargs.get("log_dir", "lightning_logs")
- # ) # logging results to a tensorboard
- # trainer = _fit(log=logger)
- # except ValueError:
- # issue with pytorch forecasting model log_prediction() function
- # pytorch-forecasting issue #1145
- trainer = _fit(log=False)
- best_model_path = trainer.checkpoint_callback.best_model_path
- best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
- train_time = time.time() - current_time
- self._model = best_tft
- return train_time
-
- def predict(self, X):
- import pandas as pd
-
- ids = self.group_ids.copy()
- ids.append(TS_TIMESTAMP_COL)
- encoder_data = self.data[lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length]
- # following pytorchforecasting example, make all target values equal to the last data
- last_data_cols = self.group_ids.copy()
- last_data_cols.append(TS_VALUE_COL)
- last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]
- decoder_data = X
- if "time_idx" not in decoder_data:
- decoder_data = add_time_idx_col(decoder_data)
- decoder_data["time_idx"] += encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()
- # decoder_data[TS_VALUE_COL] = 0
- decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)
- decoder_data = decoder_data.sort_values(ids)
- new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)
- new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")
- new_raw_predictions = self._model.predict(new_prediction_data)
- index = [decoder_data[idx].to_numpy() for idx in ids]
- predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)
- return predictions
-
-
class suppress_stdout_stderr(object):
def __init__(self):
# Open a pair of null files
diff --git a/flaml/automl/state.py b/flaml/automl/state.py
index 1597271a1..1b473b75d 100644
--- a/flaml/automl/state.py
+++ b/flaml/automl/state.py
@@ -1,11 +1,12 @@
import inspect
+import copy
import time
from typing import Any, Optional
import numpy as np
from flaml import tune
from flaml.automl.logger import logger
from flaml.automl.ml import compute_estimator, train_estimator
-from flaml.automl.task.task import TS_FORECAST
+from flaml.automl.time_series.ts_data import TimeSeriesDataset
from flaml.automl.spark import psDataFrame, psSeries, DataFrame, Series
@@ -55,7 +56,7 @@ class SearchState:
def __init__(
self,
learner_class,
- data_size,
+ data,
task,
starting_point=None,
period=None,
@@ -68,14 +69,18 @@ class SearchState:
self.init_config = None
self.low_cost_partial_config = {}
self.cat_hp_cost = {}
- self.data_size = data_size
+
self.ls_ever_converged = False
self.learner_class = learner_class
self._budget = budget
- if task in TS_FORECAST:
- search_space = learner_class.search_space(data_size=data_size, task=task, pred_horizon=period)
+
+ if task.is_ts_forecast():
+ data_size = data.train_data.shape
+ search_space = learner_class.search_space(data=data, task=task, pred_horizon=period)
else:
+ data_size = data.shape
search_space = learner_class.search_space(data_size=data_size, task=task)
+ self.data_size = data_size
if custom_hp is not None:
search_space.update(custom_hp)
@@ -211,17 +216,22 @@ class SearchState:
class AutoMLState:
- def _prepare_sample_train_data(self, sample_size: int):
+ def prepare_sample_train_data(self, sample_size: int):
sampled_weight = groups = None
if sample_size <= self.data_size[0]:
- if isinstance(self.X_train, (DataFrame, psDataFrame)):
- sampled_X_train = self.X_train.iloc[:sample_size]
+ if isinstance(self.X_train, TimeSeriesDataset):
+ sampled_X_train = copy.copy(self.X_train)
+ sampled_X_train.train_data = self.X_train.train_data.iloc[-sample_size:]
+ sampled_y_train = None
else:
- sampled_X_train = self.X_train[:sample_size]
- if isinstance(self.y_train, (Series, psSeries)):
- sampled_y_train = self.y_train.iloc[:sample_size]
- else:
- sampled_y_train = self.y_train[:sample_size]
+ if isinstance(self.X_train, (DataFrame, psDataFrame)):
+ sampled_X_train = self.X_train.iloc[:sample_size]
+ else:
+ sampled_X_train = self.X_train[:sample_size]
+ if isinstance(self.y_train, (Series, psSeries)):
+ sampled_y_train = self.y_train.iloc[:sample_size]
+ else:
+ sampled_y_train = self.y_train[:sample_size]
weight = self.fit_kwargs.get(
"sample_weight"
) # NOTE: _prepare_sample_train_data is before kwargs is updated to fit_kwargs_by_estimator
@@ -266,7 +276,7 @@ class AutoMLState:
sampled_y_train,
sampled_weight,
groups,
- ) = state._prepare_sample_train_data(sample_size)
+ ) = state.task.prepare_sample_train_data(state, sample_size)
if sampled_weight is not None:
weight = this_estimator_kwargs["sample_weight"]
this_estimator_kwargs["sample_weight"] = sampled_weight
@@ -354,7 +364,7 @@ class AutoMLState:
sampled_y_train,
sampled_weight,
groups,
- ) = self._prepare_sample_train_data(sample_size)
+ ) = self.task.prepare_sample_train_data(self, sample_size)
if sampled_weight is not None:
weight = this_estimator_kwargs[
"sample_weight"
diff --git a/flaml/automl/task/__init__.py b/flaml/automl/task/__init__.py
index e69de29bb..280e6a2ad 100644
--- a/flaml/automl/task/__init__.py
+++ b/flaml/automl/task/__init__.py
@@ -0,0 +1 @@
+from .task import Task
diff --git a/flaml/automl/task/factory.py b/flaml/automl/task/factory.py
index 7974b9366..fcb6f82d3 100644
--- a/flaml/automl/task/factory.py
+++ b/flaml/automl/task/factory.py
@@ -1,8 +1,8 @@
from typing import Optional, Union
import numpy as np
-from flaml.automl.task.generic_task import GenericTask
-from flaml.automl.task.task import Task
+
from flaml.automl.data import DataFrame, Series
+from flaml.automl.task.task import Task, TS_FORECAST
def task_factory(
@@ -10,4 +10,10 @@ def task_factory(
X_train: Optional[Union[np.ndarray, DataFrame]] = None,
y_train: Optional[Union[np.ndarray, DataFrame, Series]] = None,
) -> Task:
- return GenericTask(task_name, X_train, y_train)
+ from flaml.automl.task.generic_task import GenericTask
+ from flaml.automl.task.time_series_task import TimeSeriesTask
+
+ if task_name in TS_FORECAST:
+ return TimeSeriesTask(task_name, X_train, y_train)
+ else:
+ return GenericTask(task_name, X_train, y_train)
diff --git a/flaml/automl/task/generic_task.py b/flaml/automl/task/generic_task.py
index 7f480365b..d4c83ef86 100644
--- a/flaml/automl/task/generic_task.py
+++ b/flaml/automl/task/generic_task.py
@@ -3,21 +3,8 @@ import time
from typing import List, Optional
import numpy as np
from flaml.automl.data import TS_TIMESTAMP_COL, concat
-from flaml.automl.ml import EstimatorSubclass, default_cv_score_agg_func, get_val_loss
-from flaml.automl.model import (
- XGBoostSklearnEstimator,
- XGBoostLimitDepthEstimator,
- RandomForestEstimator,
- LGBMEstimator,
- LRL1Classifier,
- LRL2Classifier,
- CatBoostEstimator,
- ExtraTreesEstimator,
- KNeighborsEstimator,
- TransformersEstimator,
- TransformersEstimatorModelSelection,
- SparkLGBMEstimator,
-)
+from flaml.automl.ml import EstimatorSubclass, get_val_loss, default_cv_score_agg_func
+
from flaml.automl.task.task import (
Task,
get_classification_objective,
@@ -58,20 +45,40 @@ logger = logging.getLogger(__name__)
class GenericTask(Task):
- estimators = {
- "xgboost": XGBoostSklearnEstimator,
- "xgb_limitdepth": XGBoostLimitDepthEstimator,
- "rf": RandomForestEstimator,
- "lgbm": LGBMEstimator,
- "lrl1": LRL1Classifier,
- "lrl2": LRL2Classifier,
- "catboost": CatBoostEstimator,
- "extra_tree": ExtraTreesEstimator,
- "kneighbor": KNeighborsEstimator,
- "transformer": TransformersEstimator,
- "transformer_ms": TransformersEstimatorModelSelection,
- "lgbm_spark": SparkLGBMEstimator,
- }
+ @property
+ def estimators(self):
+ if self._estimators is None:
+ # put this into a function to avoid circular dependency
+ from flaml.automl.model import (
+ XGBoostSklearnEstimator,
+ XGBoostLimitDepthEstimator,
+ RandomForestEstimator,
+ LGBMEstimator,
+ LRL1Classifier,
+ LRL2Classifier,
+ CatBoostEstimator,
+ ExtraTreesEstimator,
+ KNeighborsEstimator,
+ TransformersEstimator,
+ TransformersEstimatorModelSelection,
+ SparkLGBMEstimator,
+ )
+
+ self._estimators = {
+ "xgboost": XGBoostSklearnEstimator,
+ "xgb_limitdepth": XGBoostLimitDepthEstimator,
+ "rf": RandomForestEstimator,
+ "lgbm": LGBMEstimator,
+ "lgbm_spark": SparkLGBMEstimator,
+ "lrl1": LRL1Classifier,
+ "lrl2": LRL2Classifier,
+ "catboost": CatBoostEstimator,
+ "extra_tree": ExtraTreesEstimator,
+ "kneighbor": KNeighborsEstimator,
+ "transformer": TransformersEstimator,
+ "transformer_ms": TransformersEstimatorModelSelection,
+ }
+ return self._estimators
def validate_data(
self,
@@ -233,55 +240,7 @@ class GenericTask(Task):
state.groups_val = groups_val
state.groups = groups
- @staticmethod
- def _validate_ts_data(
- dataframe,
- y_train_all=None,
- ):
- assert (
- dataframe[dataframe.columns[0]].dtype.name == "datetime64[ns]"
- ), f"For '{TS_FORECAST}' task, the first column must contain timestamp values."
- if y_train_all is not None:
- if isinstance(y_train_all, pd.Series):
- y_df = pd.DataFrame(y_train_all)
- elif isinstance(y_train_all, np.ndarray):
- y_df = pd.DataFrame(y_train_all, columns=["labels"])
- elif isinstance(y_train_all, (psDataFrame, psSeries)):
- # TODO: optimize this
- set_option("compute.ops_on_diff_frames", True)
- y_df = y_train_all
- dataframe = dataframe.join(y_df)
- duplicates = dataframe.duplicated()
- if isinstance(dataframe, psDataFrame):
- if duplicates.any():
- logger.warning("Duplicate timestamp values found in timestamp column.")
- dataframe = dataframe.drop_duplicates()
- logger.warning("Removed duplicate rows based on all columns")
- assert (
- dataframe[[dataframe.columns[0]]].duplicated().any() is False
- ), "Duplicate timestamp values with different values for other columns."
- ts_series = ps.to_datetime(dataframe[dataframe.columns[0]])
- inferred_freq = None # TODO: `pd.infer_freq()` is not implemented yet.
- else:
- if any(duplicates):
- logger.warning(
- "Duplicate timestamp values found in timestamp column. "
- f"\n{dataframe.loc[duplicates, dataframe][dataframe.columns[0]]}"
- )
- dataframe = dataframe.drop_duplicates()
- logger.warning("Removed duplicate rows based on all columns")
- assert (
- dataframe[[dataframe.columns[0]]].duplicated() is None
- ), "Duplicate timestamp values with different values for other columns."
- ts_series = pd.to_datetime(dataframe[dataframe.columns[0]])
- inferred_freq = pd.infer_freq(ts_series)
- if inferred_freq is None:
- logger.warning(
- "Missing timestamps detected. To avoid error with estimators, set estimator list to ['prophet']. "
- )
- if y_train_all is not None:
- return dataframe.iloc[:, :-1], dataframe.iloc[:, -1]
- return dataframe
+ automl.data_size_full = len(automl._y_train_all)
@staticmethod
def _split_pyspark(state, X_train_all, y_train_all, split_ratio, stratify=None):
@@ -469,85 +428,50 @@ class GenericTask(Task):
X_train, y_train = X_train_all, y_train_all
state.groups_all = state.groups
if X_val is None and eval_method == "holdout":
- # if eval_method = holdout, make holdout data
if split_type == "time":
- if self.is_ts_forecast():
- period = state.fit_kwargs[
- "period"
- ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
- if self.is_ts_forecastpanel():
- X_train_all["time_idx"] -= X_train_all["time_idx"].min()
- X_train_all["time_idx"] = X_train_all["time_idx"].astype("int")
- ids = state.fit_kwargs["group_ids"].copy()
- ids.append(TS_TIMESTAMP_COL)
- ids.append("time_idx")
- y_train_all = (
- pd.DataFrame(y_train_all)
- if not is_spark_dataframe
- else ps.DataFrame(y_train_all)
- if isinstance(y_train_all, psSeries)
- else y_train_all
- )
- y_train_all[ids] = X_train_all[ids]
- X_train_all = X_train_all.sort_values(ids)
- y_train_all = y_train_all.sort_values(ids)
- training_cutoff = X_train_all["time_idx"].max() - period
- X_train = X_train_all[X_train_all["time_idx"] <= training_cutoff]
- y_train = y_train_all[y_train_all["time_idx"] <= training_cutoff].drop(columns=ids)
- X_val = X_train_all[X_train_all["time_idx"] > training_cutoff]
- y_val = y_train_all[y_train_all["time_idx"] > training_cutoff].drop(columns=ids)
- else:
- num_samples = X_train_all.shape[0]
- assert period < num_samples, f"period={period}>#examples={num_samples}"
- split_idx = num_samples - period
- X_train = X_train_all[:split_idx]
- y_train = y_train_all[:split_idx]
- X_val = X_train_all[split_idx:]
- y_val = y_train_all[split_idx:]
+ assert not self.is_ts_forecast(), "For a TS forecast task, this code should never be called"
+
+ is_sample_weight = "sample_weight" in state.fit_kwargs
+ if not is_spark_dataframe and is_sample_weight:
+ (
+ X_train,
+ X_val,
+ y_train,
+ y_val,
+ state.fit_kwargs[
+ "sample_weight"
+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
+ state.weight_val,
+ ) = train_test_split(
+ X_train_all,
+ y_train_all,
+ state.fit_kwargs[
+ "sample_weight"
+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
+ test_size=split_ratio,
+ shuffle=False,
+ )
+ elif not is_spark_dataframe and not is_sample_weight:
+ X_train, X_val, y_train, y_val = train_test_split(
+ X_train_all,
+ y_train_all,
+ test_size=split_ratio,
+ shuffle=False,
+ )
+ elif is_spark_dataframe and is_sample_weight:
+ (
+ X_train,
+ X_val,
+ y_train,
+ y_val,
+ state.fit_kwargs[
+ "sample_weight"
+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
+ state.weight_val,
+ ) = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)
else:
- is_sample_weight = "sample_weight" in state.fit_kwargs
- if not is_spark_dataframe and is_sample_weight:
- (
- X_train,
- X_val,
- y_train,
- y_val,
- state.fit_kwargs[
- "sample_weight"
- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
- state.weight_val,
- ) = train_test_split(
- X_train_all,
- y_train_all,
- state.fit_kwargs[
- "sample_weight"
- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
- test_size=split_ratio,
- shuffle=False,
- )
- elif not is_spark_dataframe and not is_sample_weight:
- X_train, X_val, y_train, y_val = train_test_split(
- X_train_all,
- y_train_all,
- test_size=split_ratio,
- shuffle=False,
- )
- elif is_spark_dataframe and is_sample_weight:
- (
- X_train,
- X_val,
- y_train,
- y_val,
- state.fit_kwargs[
- "sample_weight"
- ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
- state.weight_val,
- ) = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)
- else:
- X_train, X_val, y_train, y_val = self._split_pyspark(
- state, X_train_all, y_train_all, split_ratio
- )
- elif split_type == "group":
+ X_train, X_val, y_train, y_val = self._split_pyspark(state, X_train_all, y_train_all, split_ratio)
+ if split_type == "group":
gss = GroupShuffleSplit(n_splits=1, test_size=split_ratio, random_state=RANDOM_SEED)
for train_idx, val_idx in gss.split(X_train_all, y_train_all, state.groups_all):
if data_is_df:
@@ -591,6 +515,7 @@ class GenericTask(Task):
state, X_train_all, y_train_all, split_ratio=split_ratio
)
state.data_size = X_train.shape
+ state.data_size_full = len(y_train_all)
state.X_train, state.y_train = X_train, y_train
state.X_val, state.y_val = X_val, y_val
state.X_train_all = X_train_all
@@ -636,6 +561,7 @@ class GenericTask(Task):
state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period * n_groups)
else:
state.kf = TimeSeriesSplit(n_splits=n_splits)
+ # state.kf = TimeSeriesSplit(n_splits=n_splits)
elif isinstance(split_type, str):
# logger.info("Using RepeatedKFold")
state.kf = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED)
@@ -653,6 +579,7 @@ class GenericTask(Task):
fit_kwargs,
groups=None,
) -> str:
+ assert not self.is_ts_forecast(), "This function should never be called as part of a time-series task."
if self.name == "classification":
self.name = get_classification_objective(len_labels(y_train_all))
if not isinstance(split_type, str):
@@ -664,20 +591,6 @@ class GenericTask(Task):
), "GroupKFold requires groups to be provided."
return split_type
- elif self.is_ts_forecast():
- assert split_type in ["auto", "time"]
- assert isinstance(
- fit_kwargs.get("period"),
- int, # NOTE: _decide_split_type is before kwargs is updated to fit_kwargs_by_estimator
- ), f"missing a required integer 'period' for '{TS_FORECAST}' task."
- if fit_kwargs.get("group_ids"):
- # TODO (MARK) This will likely not play well with the task class
- self.name = TS_FORECASTPANEL
- assert isinstance(
- fit_kwargs.get("group_ids"), list
- ), f"missing a required List[str] 'group_ids' for '{TS_FORECASTPANEL}' task."
- return "time"
-
elif self.is_classification():
assert split_type in ["auto", "stratified", "uniform", "time", "group"]
return split_type if split_type != "auto" else groups is None and "stratified" or "group"
@@ -915,23 +828,23 @@ class GenericTask(Task):
"xgb_limitdepth",
"lgbm_spark",
]
- if self.is_ts_forecast():
- # catboost is removed because it has a `name` parameter, making it incompatible with hcrystalball
- if "catboost" in estimator_list:
- estimator_list.remove("catboost")
- if self.is_ts_forecastregression():
- try:
- import prophet
-
- estimator_list += [
- "prophet",
- "arima",
- "sarimax",
- "holt-winters",
- ]
- except ImportError:
- estimator_list += ["arima", "sarimax", "holt-winters"]
- elif not self.is_regression():
+ # if self.is_ts_forecast():
+ # # catboost is removed because it has a `name` parameter, making it incompatible with hcrystalball
+ # if "catboost" in estimator_list:
+ # estimator_list.remove("catboost")
+ # if self.is_ts_forecastregression():
+ # try:
+ # import prophet
+ #
+ # estimator_list += [
+ # "prophet",
+ # "arima",
+ # "sarimax",
+ # "holt-winters",
+ # ]
+ # except ImportError:
+ # estimator_list += ["arima", "sarimax", "holt-winters"]
+ if not self.is_regression():
estimator_list += ["lrl1"]
estimator_list = [
@@ -961,3 +874,7 @@ class GenericTask(Task):
return "ndcg"
else:
return "r2"
+
+ @staticmethod
+ def prepare_sample_train_data(automlstate, sample_size):
+ return automlstate.prepare_sample_train_data(sample_size)
diff --git a/flaml/automl/task/task.py b/flaml/automl/task/task.py
index c833627b0..4b982492c 100644
--- a/flaml/automl/task/task.py
+++ b/flaml/automl/task/task.py
@@ -69,8 +69,6 @@ class Task(ABC):
operations which vary between tasks.
"""
- estimators = {}
-
def __init__(
self,
task_name: str,
@@ -88,6 +86,7 @@ class Task(ABC):
such as in binary vs multilabel classification.
"""
self.name = task_name
+ self._estimators = None
def __str__(self) -> str:
"""Name of this task type."""
@@ -326,8 +325,7 @@ class Task(ABC):
"""For backward compatibility with all the string comparisons to task"""
return self.name == other
- @classmethod
- def estimator_class_from_str(cls, estimator_name: str) -> "flaml.automl.ml.BaseEstimator":
+ def estimator_class_from_str(self, estimator_name: str) -> "flaml.automl.ml.BaseEstimator":
"""Determine the estimator class corresponding to the provided name.
Args:
@@ -339,11 +337,11 @@ class Task(ABC):
Raises:
ValueError: The provided estimator_name has not been registered for this task type.
"""
- if estimator_name in cls.estimators:
- return cls.estimators[estimator_name]
+ if estimator_name in self.estimators:
+ return self.estimators[estimator_name]
else:
raise ValueError(
f"{estimator_name} is not a built-in learner for this task type, "
- f"only {list(cls.estimators.keys())} are supported."
+ f"only {list(self.estimators.keys())} are supported."
"Please use AutoML.add_learner() to add a customized learner."
)
diff --git a/flaml/automl/task/time_series_task.py b/flaml/automl/task/time_series_task.py
new file mode 100644
index 000000000..183f6c406
--- /dev/null
+++ b/flaml/automl/task/time_series_task.py
@@ -0,0 +1,523 @@
+import logging
+import time
+from typing import List
+
+import pandas as pd
+import numpy as np
+from scipy.sparse import issparse
+from sklearn.model_selection import (
+ GroupKFold,
+ TimeSeriesSplit,
+)
+
+from flaml.automl.ml import get_val_loss, default_cv_score_agg_func
+from flaml.automl.time_series.ts_data import (
+ TimeSeriesDataset,
+ DataTransformerTS,
+ normalize_ts_data,
+)
+
+from flaml.automl.task.task import (
+ Task,
+ get_classification_objective,
+ TS_FORECAST,
+ TS_FORECASTPANEL,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TimeSeriesTask(Task):
+ @property
+ def estimators(self):
+ if self._estimators is None:
+ # put this into a function to avoid circular dependency
+ from flaml.automl.time_series import (
+ XGBoost_TS,
+ XGBoostLimitDepth_TS,
+ RF_TS,
+ LGBM_TS,
+ ExtraTrees_TS,
+ CatBoost_TS,
+ Prophet,
+ Orbit,
+ ARIMA,
+ SARIMAX,
+ TemporalFusionTransformerEstimator,
+ HoltWinters,
+ )
+
+ self._estimators = {
+ "xgboost": XGBoost_TS,
+ "xgb_limitdepth": XGBoostLimitDepth_TS,
+ "rf": RF_TS,
+ "lgbm": LGBM_TS,
+ "extra_tree": ExtraTrees_TS,
+ "arima": ARIMA,
+ "sarimax": SARIMAX,
+ "holt-winters": HoltWinters,
+ "catboost": CatBoost_TS,
+ "tft": TemporalFusionTransformerEstimator,
+ }
+
+ try:
+ from prophet import Prophet as foo
+
+ self._estimators["prophet"] = Prophet
+ except ImportError:
+ logger.info("Couldn't import Prophet, skipping")
+
+ try:
+ from orbit.models import DLT
+
+ self._estimators["orbit"] = Orbit
+ except ImportError:
+ logger.info("Couldn't import Prophet, skipping")
+
+ return self._estimators
+
+ # processed
+ def validate_data(
+ self,
+ automl,
+ state,
+ X_train_all,
+ y_train_all,
+ dataframe,
+ label,
+ X_val=None,
+ y_val=None,
+ groups_val=None,
+ groups=None,
+ ):
+ # first beat the data into a TimeSeriesDataset shape
+ if isinstance(X_train_all, TimeSeriesDataset):
+ # in this case, we're most likely being called by another FLAML instance
+ # so all the preliminary cleaning has already been done
+ pre_data = X_train_all
+ val_len = len(pre_data.X_val)
+ else:
+ if label is None and dataframe is not None:
+ raise ValueError("If data is specified via dataframe parameter, you must also specify label")
+
+ if isinstance(y_train_all, pd.Series):
+ label = y_train_all.name
+ elif isinstance(y_train_all, np.ndarray):
+ label = "y" # Prophet convention
+
+ if isinstance(label, str):
+ target_names = [label]
+ else:
+ target_names = label
+
+ if self.time_col is None:
+ if isinstance(X_train_all, pd.DataFrame):
+ assert dataframe is None, "One of dataframe and X arguments must be None"
+ self.time_col = X_train_all.columns[0]
+ elif dataframe is not None:
+ assert X_train_all is None, "One of dataframe and X arguments must be None"
+ self.time_col = dataframe.columns[0]
+ else:
+ self.time_col = "ds"
+
+ automl._df = True
+
+ if X_train_all is not None:
+ assert y_train_all is not None, "If X_train_all is not None, y_train_all must also be"
+ assert dataframe is None, "If X_train_all is provided, dataframe must be None"
+ dataframe = TimeSeriesDataset.to_dataframe(X_train_all, y_train_all, target_names, self.time_col)
+
+ elif dataframe is not None:
+ assert label is not None, "A label or list of labels must be provided."
+ assert isinstance(dataframe, pd.DataFrame), "dataframe must be a pandas DataFrame"
+ assert label in dataframe.columns, f"{label} must a column name in dataframe"
+ else:
+ raise ValueError("Must supply either X_train_all and y_train_all, or dataframe and label")
+
+ try:
+ dataframe[self.time_col] = pd.to_datetime(dataframe[self.time_col])
+ except Exception:
+ raise ValueError(
+ f"For '{TS_FORECAST}' task, time column {self.time_col} must contain timestamp values."
+ )
+
+ dataframe = remove_ts_duplicates(dataframe, self.time_col)
+
+ if X_val is not None:
+ assert y_val is not None, "If X_val is not None, y_val must also be"
+ val_df = TimeSeriesDataset.to_dataframe(X_val, y_val, target_names, self.time_col)
+ val_len = len(val_df)
+ else:
+ val_len = 0
+ val_df = None
+
+ pre_data = TimeSeriesDataset(
+ train_data=dataframe,
+ time_col=self.time_col,
+ target_names=target_names,
+ test_data=val_df,
+ )
+
+ # TODO: should the transformer be a property of the dataset instead?
+ automl._transformer = DataTransformerTS(self.time_col, label)
+ Xt, yt = automl._transformer.fit_transform(pre_data.X_all, pre_data.y_all)
+
+ df_t = pd.concat([Xt, yt], axis=1)
+
+ data = TimeSeriesDataset(
+ train_data=df_t,
+ time_col=pre_data.time_col,
+ target_names=pre_data.target_names,
+ ).move_validation_boundary(-val_len)
+
+ # now setup the properties of all the other relevant objects
+
+ # TODO: where are these used? Replace with pointers to data?
+ automl._X_train_all, automl._y_train_all = Xt, yt
+
+ # TODO: where are these used?
+ automl._nrow, automl._ndim = data.X_train.shape
+
+ # make a property instead? Or just fix the call?
+ automl._label_transformer = automl._transformer.label_transformer
+
+ automl._feature_names_in_ = (
+ automl._X_train_all.columns.to_list() if hasattr(automl._X_train_all, "columns") else None
+ )
+
+ self.time_col = data.time_col
+ self.target_names = data.target_names
+
+ automl._state.X_val = data
+ automl._state.X_train = data
+ automl._state.y_train = None
+ automl._state.y_val = None
+ if data.test_data is not None and len(data.test_data) > 0:
+ automl._state.X_train_all = data.move_validation_boundary(len(data.test_data))
+ else:
+ automl._state.X_train_all = data
+ automl._state.y_train_all = None
+
+ automl._state.data_size = data.train_data.shape
+ automl.data_size_full = len(data.all_data)
+ automl._state.groups = None
+ automl._sample_weight_full = None
+
+ def prepare_data(
+ self,
+ state,
+ X_train_all,
+ y_train_all,
+ auto_argument,
+ eval_method,
+ split_type,
+ split_ratio,
+ n_splits,
+ data_is_df,
+ sample_weight_full,
+ time_col=None,
+ ):
+ state.kf = None
+ state.data_size_full = len(y_train_all)
+
+ if split_type in ["uniform", "stratified"]:
+ raise ValueError(f"Split type {split_type} is not valid for time series")
+
+ state.groups = None
+ state.groups_all = None
+ state.groups_val = None
+
+ ts_data = state.X_val
+ no_test_data = ts_data is None or ts_data.test_data is None or len(ts_data.test_data) == 0
+ if no_test_data and eval_method == "holdout":
+ # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
+ period = state.fit_kwargs["period"]
+
+ if self.name == TS_FORECASTPANEL:
+ # TODO: move this into the TimeSeriesDataset class
+ X_train_all = ts_data.X_train
+ y_train_all = ts_data.y_train
+
+ X_train_all["time_idx"] -= X_train_all["time_idx"].min()
+ X_train_all["time_idx"] = X_train_all["time_idx"].astype("int")
+ ids = state.fit_kwargs["group_ids"].copy()
+ ids.append(ts_data.time_col)
+ ids.append("time_idx")
+ y_train_all = pd.DataFrame(y_train_all)
+ y_train_all[ids] = X_train_all[ids]
+ X_train_all = X_train_all.sort_values(ids)
+ y_train_all = y_train_all.sort_values(ids)
+ training_cutoff = X_train_all["time_idx"].max() - period
+ X_train = X_train_all[lambda x: x.time_idx <= training_cutoff]
+ y_train = y_train_all[lambda x: x.time_idx <= training_cutoff].drop(columns=ids)
+ X_val = X_train_all[lambda x: x.time_idx > training_cutoff]
+ y_val = y_train_all[lambda x: x.time_idx > training_cutoff].drop(columns=ids)
+
+ train_data = normalize_ts_data(
+ X_train,
+ ts_data.target_names,
+ ts_data.time_col,
+ y_train,
+ )
+ test_data = normalize_ts_data(
+ X_val,
+ ts_data.target_names,
+ ts_data.time_col,
+ y_val,
+ )
+ ts_data = TimeSeriesDataset(
+ train_data,
+ ts_data.time_col,
+ ts_data.target_names,
+ ts_data.frequency,
+ test_data,
+ )
+ state.X_val = ts_data
+ state.X_train = ts_data
+
+ else:
+ # if eval_method = holdout, make holdout data
+ num_samples = ts_data.train_data.shape[0]
+ assert period < num_samples, f"period={period}>#examples={num_samples}"
+ state.X_val = ts_data.move_validation_boundary(-period)
+ state.X_train = state.X_val
+
+ if eval_method != "holdout":
+ if self.name != TS_FORECASTPANEL:
+ period = state.fit_kwargs[
+ "period"
+ ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
+ step_size = state.fit_kwargs.get("cv_step_size", period)
+
+ ts_data = state.X_train
+ if n_splits * step_size + 2 * period > ts_data.y_train.size:
+ n_splits = int((ts_data.y_train.size - 2 * period) / step_size)
+ assert n_splits >= 2, (
+ f"cross validation for forecasting period={period}"
+ f" requires input data with at least {2*period + 2*step_size} examples."
+ )
+ logger.info(f"Using nsplits={n_splits} due to data size limit.")
+ state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period)
+ state.kf.step_size = step_size
+
+ else:
+ n_groups = ts_data.X_train.groupby(state.fit_kwargs.get("group_ids")).ngroups
+ period = state.fit_kwargs["period"]
+ state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period * n_groups)
+
+ # TODO: move task detection to Task.__init__!
+ def decide_split_type(
+ self,
+ split_type,
+ y_train_all,
+ fit_kwargs,
+ groups=None,
+ ) -> str:
+ # TODO: move into task creation!!!
+ if self.name == "classification":
+ self.name = get_classification_objective(len(np.unique(y_train_all)))
+
+ # TODO: do we need this?
+ if not isinstance(split_type, str):
+ assert hasattr(split_type, "split") and hasattr(
+ split_type, "get_n_splits"
+ ), "split_type must be a string or a splitter object with split and get_n_splits methods."
+ assert (
+ not isinstance(split_type, GroupKFold) or groups is not None
+ ), "GroupKFold requires groups to be provided."
+ return split_type
+
+ else:
+ assert split_type in ["auto", "time"]
+ assert isinstance(
+ fit_kwargs.get("period"),
+ int, # NOTE: _decide_split_type is before kwargs is updated to fit_kwargs_by_estimator
+ ), f"missing a required integer 'period' for '{TS_FORECAST}' task."
+ if fit_kwargs.get("group_ids"):
+ # TODO (MARK) This will likely not play well with the task class
+ self.name = TS_FORECASTPANEL
+ assert isinstance(
+ fit_kwargs.get("group_ids"), list
+ ), f"missing a required List[str] 'group_ids' for '{TS_FORECASTPANEL}' task."
+ return "time"
+
+ # TODO: merge with preprocess() below
+ def _preprocess(self, X, transformer=None):
+ if isinstance(X, List):
+ try:
+ if isinstance(X[0], List):
+ X = [x for x in zip(*X)]
+ X = pd.DataFrame(
+ dict(
+ [
+ (transformer._str_columns[idx], X[idx])
+ if isinstance(X[0], List)
+ else (transformer._str_columns[idx], [X[idx]])
+ for idx in range(len(X))
+ ]
+ )
+ )
+ except IndexError:
+ raise IndexError("Test data contains more columns than training data, exiting")
+ elif isinstance(X, int):
+ return X
+ elif issparse(X):
+ X = X.tocsr()
+ if self.is_ts_forecast():
+ X = pd.DataFrame(X)
+ if transformer:
+ X = transformer.transform(X)
+ return X
+
+ def preprocess(self, X, transformer=None):
+ if isinstance(X, pd.DataFrame) or isinstance(X, np.ndarray) or isinstance(X, pd.Series):
+ X = X.copy()
+ X = normalize_ts_data(X, self.target_names, self.time_col)
+ return self._preprocess(X, transformer)
+ elif isinstance(X, int):
+ return X
+ else:
+ raise ValueError(f"unknown type of X, {X.__class__}")
+
+ def evaluate_model_CV(
+ self,
+ config,
+ estimator,
+ X_train_all,
+ y_train_all,
+ budget,
+ kf,
+ eval_metric,
+ best_val_loss,
+ cv_score_agg_func=None,
+ log_training_metric=False,
+ fit_kwargs={},
+ free_mem_ratio=0, # what is this for?
+ ):
+ if cv_score_agg_func is None:
+ cv_score_agg_func = default_cv_score_agg_func
+ start_time = time.time()
+ val_loss_folds = []
+ log_metric_folds = []
+ metric = None
+ train_time = pred_time = 0
+ total_fold_num = 0
+ n = kf.get_n_splits()
+ if self.is_classification():
+ labels = np.unique(y_train_all)
+ else:
+ labels = fit_kwargs.get("label_list") # pass the label list on to compute the evaluation metric
+ ts_data = X_train_all
+ budget_per_train = budget / n
+ ts_data = X_train_all
+ for data in ts_data.cv_train_val_sets(kf.n_splits, kf.test_size, kf.step_size):
+ estimator.cleanup()
+ val_loss_i, metric_i, train_time_i, pred_time_i = get_val_loss(
+ config,
+ estimator,
+ X_train=data,
+ y_train=None,
+ X_val=data,
+ y_val=None,
+ eval_metric=eval_metric,
+ labels=labels,
+ budget=budget_per_train,
+ log_training_metric=log_training_metric,
+ fit_kwargs=fit_kwargs,
+ task=self,
+ weight_val=None,
+ groups_val=None,
+ free_mem_ratio=free_mem_ratio,
+ )
+ if isinstance(metric_i, dict) and "intermediate_results" in metric_i:
+ del metric_i["intermediate_results"]
+ total_fold_num += 1
+ val_loss_folds.append(val_loss_i)
+ log_metric_folds.append(metric_i)
+ train_time += train_time_i
+ pred_time += pred_time_i
+ if time.time() - start_time >= budget:
+ break
+ val_loss, metric = cv_score_agg_func(val_loss_folds, log_metric_folds)
+ n = total_fold_num
+ pred_time /= n
+ return val_loss, metric, train_time, pred_time
+
+ def default_estimator_list(self, estimator_list: List[str], is_spark_dataframe: bool) -> List[str]:
+ assert not is_spark_dataframe, "Spark is not yet supported for time series"
+
+ # TODO: why not do this if/then in the calling function?
+ if "auto" != estimator_list:
+ return estimator_list
+
+ if self.is_ts_forecastpanel():
+ return ["tft"]
+
+ estimator_list = [
+ "lgbm",
+ "rf",
+ "xgboost",
+ "extra_tree",
+ "xgb_limitdepth",
+ ]
+
+ # Catboost appears to be way slower than the others, don't include it by default
+ # try:
+ # import catboost
+ #
+ # estimator_list.append("catboost")
+ # except ImportError:
+ # pass
+
+ if self.is_regression():
+ estimator_list += ["arima", "sarimax"]
+
+ try:
+ import prophet
+
+ estimator_list.append("prophet")
+ except ImportError:
+ pass
+
+ return estimator_list
+
+ def default_metric(self, metric: str) -> str:
+ assert self.is_ts_forecast(), "If this is not a TS forecasting task, this code should never have been called"
+ if metric == "auto":
+ return "mape"
+ else:
+ return metric
+
+ @staticmethod
+ def prepare_sample_train_data(automlstate, sample_size):
+ # we take the tail, rather than the head, for compatibility with time series
+
+ shift = sample_size - len(automlstate.X_train.train_data)
+ sampled_X_train = automlstate.X_train.move_validation_boundary(shift)
+
+ return sampled_X_train, None, None, None
+
+
+def remove_ts_duplicates(
+ X,
+ time_col,
+):
+ """
+ Assumes the targets are included
+ @param X:
+ @param time_col:
+ @param y:
+ @return:
+ """
+
+ duplicates = X.duplicated()
+
+ if any(duplicates):
+ logger.warning("Duplicate timestamp values found in timestamp column. " f"\n{X.loc[duplicates, X][time_col]}")
+ X = X.drop_duplicates()
+ logger.warning("Removed duplicate rows based on all columns")
+ assert (
+ X[[X.columns[0]]].duplicated() is None
+ ), "Duplicate timestamp values with different values for other columns."
+
+ return X
diff --git a/flaml/automl/time_series/__init__.py b/flaml/automl/time_series/__init__.py
new file mode 100644
index 000000000..0cf1c1c87
--- /dev/null
+++ b/flaml/automl/time_series/__init__.py
@@ -0,0 +1,17 @@
+from .ts_model import (
+ Prophet,
+ Orbit,
+ ARIMA,
+ SARIMAX,
+ HoltWinters,
+ LGBM_TS,
+ XGBoost_TS,
+ RF_TS,
+ ExtraTrees_TS,
+ XGBoostLimitDepth_TS,
+ CatBoost_TS,
+ TimeSeriesEstimator,
+)
+from .tft import TemporalFusionTransformerEstimator
+
+from .ts_data import TimeSeriesDataset
diff --git a/flaml/automl/time_series/feature.py b/flaml/automl/time_series/feature.py
new file mode 100644
index 000000000..8cf6eb430
--- /dev/null
+++ b/flaml/automl/time_series/feature.py
@@ -0,0 +1,34 @@
+import math
+import datetime
+from functools import lru_cache
+
+import pandas as pd
+
+
+def monthly_fourier_features(timestamps: pd.Series, month_fourier_degree: int = 2):
+ if len(timestamps):
+ data = pd.DataFrame({"time": timestamps})
+ month_pos = timestamps.apply(lambda x: position_in_month(datetime.date(x.year, x.month, x.day)))
+ for d in range(month_fourier_degree):
+ data[f"cos{d+1}"] = (2 * (d + 1) * math.pi * month_pos).apply(math.cos)
+ data[f"sin{d + 1}"] = (2 * (d + 1) * math.pi * month_pos).apply(math.sin)
+
+ drop_cols = ["time"]
+ data = data.drop(columns=drop_cols)
+ return data
+ else:
+ columns = []
+ for d in range(month_fourier_degree):
+ columns += [f"cos{d+1}", f"sin{d + 1}"]
+
+ return pd.DataFrame(columns=columns)
+
+
+@lru_cache(maxsize=4096)
+def position_in_month(d: datetime.date):
+ prev = datetime.date(d.year, d.month, 1) - datetime.timedelta(days=1)
+ nxt = datetime.date(
+ d.year + 1 if d.month == 12 else d.year, 1 if d.month == 12 else d.month + 1, 1
+ ) - datetime.timedelta(days=1)
+ delta = (d - prev).days / (nxt - prev).days
+ return delta
diff --git a/flaml/automl/time_series/sklearn.py b/flaml/automl/time_series/sklearn.py
new file mode 100644
index 000000000..175cef848
--- /dev/null
+++ b/flaml/automl/time_series/sklearn.py
@@ -0,0 +1,156 @@
+try:
+ import pandas as pd
+ from pandas import DataFrame, Series, to_datetime
+except ImportError:
+
+ class PD:
+ pass
+
+ pd = PD()
+ pd.DataFrame = None
+ pd.Series = None
+ DataFrame = Series = None
+
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.decomposition import PCA
+
+
+def make_lag_features(X: pd.DataFrame, y: pd.Series, lags: int):
+ """Transform input data X, y into autoregressive form - shift
+ them appropriately based on horizon and create `lags` columns.
+
+ Parameters
+ ----------
+ X : pandas.DataFrame
+ Input features.
+
+ y : array_like, (1d)
+ Target vector.
+
+ horizon : int
+ length of X for `predict` method
+
+ Returns
+ -------
+ pandas.DataFrame
+ shifted dataframe with `lags` columns
+ """
+ lag_features = []
+
+ # make sure we show y's _previous_ value to exclude data leaks
+ X = X.reset_index(drop=True)
+ X["lag_" + y.name] = y.shift(1).values
+
+ X_lag = X.copy()
+ for i in range(0, lags):
+ X_lag.columns = [f"{c}_lag_{i}" for c in X.columns]
+ lag_features.append(X_lag)
+ X_lag = X_lag.shift(1)
+
+ X_lags = pd.concat(lag_features, axis=1)
+ X_out = X_lags.dropna().reset_index(drop=True)
+ assert len(X_out) + lags == len(X)
+ return X_out
+
+
+class SklearnWrapper:
+ def __init__(
+ self,
+ model_class: type,
+ horizon: int,
+ lags: int,
+ init_params: dict = None,
+ fit_params: dict = None,
+ pca_features: bool = False,
+ ):
+ init_params = init_params if init_params else {}
+ self.fit_params = fit_params if fit_params else {}
+ self.lags = lags
+ self.horizon = horizon
+ # TODO: use multiregression where available
+ self.models = [model_class(**init_params) for _ in range(horizon)]
+ self.pca_features = pca_features
+ if self.pca_features:
+ self.norm = StandardScaler()
+ self.pca = None
+
+ def fit(self, X: pd.DataFrame, y: pd.Series, **kwargs):
+ self._X = X
+ self._y = y
+
+ fit_params = {**self.fit_params, **kwargs}
+ X_feat = make_lag_features(X, y, self.lags)
+ if self.pca_features:
+ X_trans = self.norm.fit_transform(X_feat)
+
+ cum_expl_var = np.cumsum(PCA(svd_solver="full").fit(X_trans).explained_variance_ratio_)
+ self.pca = PCA(svd_solver="full", n_components=np.argmax(1 - cum_expl_var < 1e-6))
+ X_trans = self.pca.fit_transform(X_trans)
+ else:
+ X_trans = X_feat
+
+ for i, model in enumerate(self.models):
+ offset = i + self.lags
+ model.fit(X_trans[: len(X) - offset], y[offset:], **fit_params)
+ return self
+
+ def predict(self, X, X_train=None, y_train=None):
+ if X_train is None:
+ X_train = self._X
+ if y_train is None:
+ y_train = self._y
+
+ X_train = X_train.reset_index(drop=True)
+ X_train[self._y.name] = y_train.values
+ Xall = pd.concat([X_train, X], axis=0).reset_index(drop=True)
+ y = Xall.pop(self._y.name)
+
+ X_feat = make_lag_features(Xall[: len(X_train) + 1], y[: len(X_train) + 1], self.lags)
+ if self.pca_features:
+ X_trans = self.pca.transform(self.norm.transform(X_feat))
+ else:
+ X_trans = X_feat
+ # predict all horizons from the latest features vector
+ preds = pd.Series([m.predict(X_trans[-1:])[0] for m in self.models])
+ if len(preds) < len(X):
+ # recursive call if len(X) > trained horizon
+ y_train = pd.concat([y_train, preds], axis=0, ignore_index=True)
+ preds = pd.concat(
+ [
+ preds,
+ self.predict(
+ X=Xall[len(y_train) :],
+ X_train=Xall[: len(y_train)],
+ y_train=y_train,
+ ),
+ ],
+ axis=0,
+ ignore_index=True,
+ )
+ if len(preds) > len(X):
+ preds = preds[: len(X)]
+
+ preds.index = X.index
+ # TODO: do we want auto-clipping?
+ # return self._clip_predictions(preds)
+ return preds
+
+ # TODO: fix
+ # @staticmethod
+ # def _adjust_holidays(X):
+ # """Transform 'holiday' columns to binary feature.
+ #
+ # Parameters
+ # ----------
+ # X : pandas.DataFrame
+ # Input features with 'holiday' column.
+ #
+ # Returns
+ # -------
+ # pandas.DataFrame
+ # Holiday feature in numeric form
+ # """
+ # return X.assign(
+ # **{col: X[col] != "" for col in X.filter(like="_holiday_").columns}
+ # )
diff --git a/flaml/automl/time_series/tft.py b/flaml/automl/time_series/tft.py
new file mode 100644
index 000000000..11a5714d9
--- /dev/null
+++ b/flaml/automl/time_series/tft.py
@@ -0,0 +1,183 @@
+import time
+
+try:
+ import pandas as pd
+ from pandas import DataFrame, Series, to_datetime
+except ImportError:
+
+ class PD:
+ pass
+
+ pd = PD()
+ pd.DataFrame = None
+ pd.Series = None
+ DataFrame = Series = None
+
+from flaml import tune
+from flaml.automl.data import add_time_idx_col
+from flaml.automl.time_series.ts_data import TimeSeriesDataset
+from flaml.automl.time_series.ts_model import TimeSeriesEstimator
+
+
+class TemporalFusionTransformerEstimator(TimeSeriesEstimator):
+ """The class for tuning Temporal Fusion Transformer"""
+
+ @classmethod
+ def search_space(cls, data, task, pred_horizon, **params):
+ space = {
+ "gradient_clip_val": {
+ "domain": tune.loguniform(lower=0.01, upper=100.0),
+ "init_value": 0.01,
+ },
+ "hidden_size": {
+ "domain": tune.lograndint(lower=8, upper=512),
+ "init_value": 16,
+ },
+ "hidden_continuous_size": {
+ "domain": tune.randint(lower=1, upper=65),
+ "init_value": 8,
+ },
+ "attention_head_size": {
+ "domain": tune.randint(lower=1, upper=5),
+ "init_value": 4,
+ },
+ "dropout": {
+ "domain": tune.uniform(lower=0.1, upper=0.3),
+ "init_value": 0.1,
+ },
+ "learning_rate": {
+ "domain": tune.loguniform(lower=0.00001, upper=1.0),
+ "init_value": 0.001,
+ },
+ }
+ return space
+
+ def transform_ds(self, X_train: TimeSeriesDataset, y_train, **kwargs):
+ self.data = X_train.train_data
+
+ max_prediction_length = kwargs["period"]
+ self.max_encoder_length = kwargs["max_encoder_length"]
+ training_cutoff = self.data["time_idx"].max() - max_prediction_length
+
+ from pytorch_forecasting import TimeSeriesDataSet
+ from pytorch_forecasting.data import GroupNormalizer
+
+ self.group_ids = kwargs["group_ids"].copy()
+ training = TimeSeriesDataSet(
+ self.data[lambda x: x.time_idx <= training_cutoff],
+ time_idx="time_idx",
+ target=X_train.target_names[0],
+ group_ids=self.group_ids,
+ min_encoder_length=kwargs.get(
+ "min_encoder_length", self.max_encoder_length // 2
+ ), # keep encoder length long (as it is in the validation set)
+ max_encoder_length=self.max_encoder_length,
+ min_prediction_length=1,
+ max_prediction_length=max_prediction_length,
+ static_categoricals=kwargs.get("static_categoricals", []),
+ static_reals=kwargs.get("static_reals", []),
+ time_varying_known_categoricals=kwargs.get("time_varying_known_categoricals", []),
+ time_varying_known_reals=kwargs.get("time_varying_known_reals", []),
+ time_varying_unknown_categoricals=kwargs.get("time_varying_unknown_categoricals", []),
+ time_varying_unknown_reals=kwargs.get("time_varying_unknown_reals", []),
+ variable_groups=kwargs.get(
+ "variable_groups", {}
+ ), # group of categorical variables can be treated as one variable
+ lags=kwargs.get("lags", {}),
+ target_normalizer=GroupNormalizer(
+ groups=kwargs["group_ids"], transformation="softplus"
+ ), # use softplus and normalize by group
+ add_relative_time_idx=True,
+ add_target_scales=True,
+ add_encoder_length=True,
+ )
+
+ # create validation set (predict=True) which means to predict the last max_prediction_length points in time
+ # for each series
+ validation = TimeSeriesDataSet.from_dataset(training, self.data, predict=True, stop_randomization=True)
+
+ # create dataloaders for model
+ batch_size = kwargs.get("batch_size", 64)
+ train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
+ val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size * 10, num_workers=0)
+
+ return training, train_dataloader, val_dataloader
+
+ def fit(self, X_train, y_train, budget=None, **kwargs):
+ import warnings
+ import pytorch_lightning as pl
+ import torch
+ from pytorch_forecasting import TemporalFusionTransformer
+ from pytorch_forecasting.metrics import QuantileLoss
+ from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
+ from pytorch_lightning.loggers import TensorBoardLogger
+
+ # a bit of monkey patching to fix the MacOS test
+ # all the log_prediction method appears to do is plot stuff, which ?breaks github tests
+ def log_prediction(*args, **kwargs):
+ pass
+
+ TemporalFusionTransformer.log_prediction = log_prediction
+
+ warnings.filterwarnings("ignore")
+ current_time = time.time()
+ super().fit(X_train, **kwargs)
+ training, train_dataloader, val_dataloader = self.transform_ds(X_train, y_train, **kwargs)
+ params = self.params.copy()
+ gradient_clip_val = params.pop("gradient_clip_val", None)
+ params.pop("n_jobs", None)
+ max_epochs = kwargs.get("max_epochs", 20)
+ early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")
+ lr_logger = LearningRateMonitor() # log the learning rate
+ logger = TensorBoardLogger(kwargs.get("log_dir", "lightning_logs")) # logging results to a tensorboard
+ default_trainer_kwargs = dict(
+ gpus=self._kwargs.get("gpu_per_trial", [0]) if torch.cuda.is_available() else None,
+ max_epochs=max_epochs,
+ gradient_clip_val=gradient_clip_val,
+ callbacks=[lr_logger, early_stop_callback],
+ logger=logger,
+ )
+ trainer = pl.Trainer(
+ **default_trainer_kwargs,
+ )
+ tft = TemporalFusionTransformer.from_dataset(
+ training,
+ **params,
+ lstm_layers=2, # 2 is mostly optimal according to documentation
+ output_size=7, # 7 quantiles by default
+ loss=QuantileLoss(),
+ log_interval=10, # uncomment for learning rate finder and otherwise, e.g. to 10 for logging every 10 batches
+ reduce_on_plateau_patience=4,
+ )
+ # fit network
+ trainer.fit(
+ tft,
+ train_dataloaders=train_dataloader,
+ val_dataloaders=val_dataloader,
+ )
+ best_model_path = trainer.checkpoint_callback.best_model_path
+ best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
+ train_time = time.time() - current_time
+ self._model = best_tft
+ return train_time
+
+ def predict(self, X):
+ ids = self.group_ids.copy()
+ ids.append(self.time_col)
+ encoder_data = self.data[lambda x: x.time_idx > x.time_idx.max() - self.max_encoder_length]
+ # following pytorchforecasting example, make all target values equal to the last data
+ last_data_cols = self.group_ids.copy()
+ last_data_cols.append(self.target_names[0])
+ last_data = self.data[lambda x: x.time_idx == x.time_idx.max()][last_data_cols]
+ decoder_data = X.X_val if isinstance(X, TimeSeriesDataset) else X
+ if "time_idx" not in decoder_data:
+ decoder_data = add_time_idx_col(decoder_data)
+ decoder_data["time_idx"] += encoder_data["time_idx"].max() + 1 - decoder_data["time_idx"].min()
+ decoder_data = decoder_data.merge(last_data, how="inner", on=self.group_ids)
+ decoder_data = decoder_data.sort_values(ids)
+ new_prediction_data = pd.concat([encoder_data, decoder_data], ignore_index=True)
+ new_prediction_data["time_idx"] = new_prediction_data["time_idx"].astype("int")
+ new_raw_predictions = self._model.predict(new_prediction_data)
+ index = [decoder_data[idx].to_numpy() for idx in ids]
+ predictions = pd.Series(new_raw_predictions.numpy().ravel(), index=index)
+ return predictions
diff --git a/flaml/automl/time_series/ts_data.py b/flaml/automl/time_series/ts_data.py
new file mode 100644
index 000000000..2dc7922a1
--- /dev/null
+++ b/flaml/automl/time_series/ts_data.py
@@ -0,0 +1,544 @@
+import copy
+import datetime
+import math
+from dataclasses import dataclass, field
+from typing import List, Optional, Callable, Dict, Generator, Union
+
+import numpy as np
+
+try:
+ import pandas as pd
+ from pandas import DataFrame, Series, to_datetime
+ from scipy.sparse import issparse
+ from sklearn.preprocessing import LabelEncoder
+ from sklearn.impute import SimpleImputer
+ from sklearn.compose import ColumnTransformer
+
+ from .feature import monthly_fourier_features
+except ImportError:
+
+ class PD:
+ pass
+
+ pd = PD()
+ pd.DataFrame = None
+ pd.Series = None
+ DataFrame = Series = None
+
+
+@dataclass
+class TimeSeriesDataset:
+ train_data: pd.DataFrame
+ time_idx: str
+ time_col: str
+ target_names: List[str]
+ frequency: str
+ test_data: pd.DataFrame
+ time_varying_known_categoricals: List[str] = field(default_factory=lambda: [])
+ time_varying_known_reals: List[str] = field(default_factory=lambda: [])
+ time_varying_unknown_categoricals: List[str] = field(default_factory=lambda: [])
+ time_varying_unknown_reals: List[str] = field(default_factory=lambda: [])
+
+ def __init__(
+ self,
+ train_data: pd.DataFrame,
+ time_col: str,
+ target_names: Union[str, List[str]],
+ time_idx: str = "time_idx",
+ test_data: Optional[pd.DataFrame] = None,
+ ):
+ self.train_data = train_data
+ self.time_col = time_col
+ self.time_idx = time_idx
+ self.target_names = [target_names] if isinstance(target_names, str) else list(target_names)
+ assert isinstance(self.target_names, list)
+ assert len(self.target_names)
+
+ self.frequency = pd.infer_freq(train_data[time_col].unique())
+ assert self.frequency is not None, "Only time series of regular frequency are currently supported."
+
+ float_cols = list(train_data.select_dtypes(include=["floating"]).columns)
+ self.time_varying_known_reals = list(set(float_cols) - set(self.target_names))
+
+ self.time_varying_known_categoricals = list(
+ set(train_data.columns) - set(self.time_varying_known_reals) - set(self.target_names) - {time_col}
+ )
+ if test_data is not None:
+ self.test_data = test_data
+ else:
+ self.test_data = pd.DataFrame(columns=self.train_data.columns)
+
+ def add_test_data(self, X: pd.DataFrame) -> "TimeSeriesDataset":
+ assert self.time_col in X.columns
+ train_data = self.all_data[self.all_data[self.time_col] < X[self.time_col].min()]
+ return TimeSeriesDataset(train_data, self.time_col, self.target_names, self.time_idx, X)
+
+ @staticmethod
+ def to_dataframe(X, y, target_names: List[str], time_col: str):
+ assert len(X) == len(y), "X_val and y_val must have the same length"
+ validate_data_basic(X, y)
+ # coerce them into a dataframe
+ val_df = normalize_ts_data(X, target_names, time_col, y)
+ return val_df
+
+ @property
+ def all_data(self):
+ if len(self.test_data):
+ return pd.concat([self.train_data, self.test_data], axis=0)
+ else:
+ return self.train_data
+
+ @property
+ def regressors(self):
+ return self.time_varying_known_categoricals + self.time_varying_known_reals
+
+ @property
+ def end_date(self):
+ test_len = 0 if self.test_data is None else len(self.test_data)
+ data = self.test_data if test_len else self.train_data
+ return data.iloc[-1][self.time_col]
+
+ def _X(self, df: pd.DataFrame):
+ features = [col for col in df.columns if col not in self.target_names]
+ return df[features]
+
+ def _y(self, df: pd.DataFrame):
+ if len(self.target_names) > 1:
+ return df[self.target_names]
+ else:
+ return df[self.target_names[0]]
+
+ @property
+ def X_train(self) -> pd.DataFrame:
+ return self._X(self.train_data)
+
+ @property
+ def X_val(self) -> pd.DataFrame:
+ return self._X(self.test_data)
+
+ @property
+ def X_all(self) -> pd.DataFrame:
+ return pd.concat([self.X_train, self.X_val], axis=0)
+
+ @property
+ def y_train(self) -> pd.DataFrame:
+ return self._y(self.train_data)
+
+ @property
+ def y_val(self) -> pd.DataFrame:
+ return self._y(self.test_data)
+
+ @property
+ def y_all(self) -> pd.DataFrame:
+ return self._y(self.all_data)
+
+ def next_scale(self) -> int:
+ scale_map = {"D": 7, "MS": 12}
+ return scale_map.get(self.frequency, 8)
+
+ def known_features_to_floats(self, train: bool, drop_first: bool = True) -> np.ndarray:
+ # this is a bit tricky as shapes for train and test data must match, so need to encode together
+ combined = pd.concat(
+ [
+ self.train_data,
+ self.test_data,
+ ],
+ ignore_index=True,
+ )
+
+ cat_one_hots = pd.get_dummies(
+ combined[self.time_varying_known_categoricals],
+ columns=self.time_varying_known_categoricals,
+ drop_first=drop_first,
+ ).values.astype(float)
+
+ reals = combined[self.time_varying_known_reals].values.astype(float)
+ both = np.concatenate([reals, cat_one_hots], axis=1)
+
+ if train:
+ return both[: len(self.train_data)]
+ else:
+ return both[len(self.train_data) :]
+
+ # def unique_dimension_values(self) -> np.ndarray:
+ # # this is the same set for train and test data, by construction
+ # return self.combine_dims(self.train_data).unique()
+ #
+ # def combine_dims(self, df):
+ # return df.apply(lambda row: tuple([row[d] for d in self.dimensions]), axis=1)
+
+ def to_univariate(self) -> Dict[str, "TimeSeriesDataset"]:
+ """
+ Convert a multivariate TrainingData to a dict of univariate ones
+ @param df:
+ @return:
+ """
+
+ train_dims = self.combine_dims(self.train_data)
+ test_dims = self.combine_dims(self.test_data)
+
+ out = {}
+ for d in train_dims.unique():
+ out[d] = copy.copy(self)
+ out[d].train_data = self.train_data[train_dims == d]
+ out[d].test_data = self.test_data[test_dims == d]
+ return out
+
+ def move_validation_boundary(self, steps: int) -> "TimeSeriesDataset":
+ out = copy.copy(self)
+ if steps > 0:
+ out.train_data = pd.concat([self.train_data, self.test_data[:steps]])
+ out.test_data = self.test_data[steps:]
+ elif steps < 0:
+ out.train_data = self.train_data[:steps]
+ if len(self.test_data):
+ out.test_data = pd.concat([self.train_data[steps:], self.test_data])
+ else:
+ out.test_data = self.train_data[steps:]
+
+ return out
+
+ def cv_train_val_sets(
+ self, n_splits: int, val_length: int, step_size: int
+ ) -> Generator["TimeSeriesDataset", None, None]:
+ max_index = len(self.train_data) - 1
+ for i in range(n_splits):
+ out = copy.copy(self)
+ val_start = max_index - (n_splits - i - 1) * step_size - val_length
+ out.train_data = self.train_data[:val_start]
+ out.test_data = self.train_data[val_start : val_start + val_length]
+ yield out
+
+ def filter(self, filter_fun: Callable) -> "TimeSeriesDataset":
+ if filter_fun is None:
+ return self
+ out = copy.copy(self)
+ out.train_data = self.train_data[filter_fun]
+ out.test_data = self.test_data[filter_fun]
+ return out
+
+ def prettify_prediction(self, y_pred: Union[pd.DataFrame, pd.Series, np.ndarray]):
+ if self.test_data is not None and len(self.test_data):
+ assert len(y_pred) == len(self.test_data)
+
+ if isinstance(y_pred, np.ndarray):
+ y_pred = pd.DataFrame(data=y_pred, columns=self.target_names, index=self.test_data.index)
+ elif isinstance(y_pred, pd.Series):
+ assert len(self.target_names) == 1, "Not enough columns in y_pred"
+ y_pred.name = self.target_names[0]
+ y_pred = pd.DataFrame(y_pred)
+ y_pred.index = self.test_data.index
+ elif isinstance(y_pred, pd.DataFrame):
+ y_pred.index = self.test_data.index
+
+ if self.time_col not in y_pred.columns:
+ y_pred[self.time_col] = self.test_data[self.time_col]
+
+ else:
+ if isinstance(y_pred, np.ndarray):
+ raise ValueError("Can't enrich np.ndarray as self.test_data is None")
+ elif isinstance(y_pred, pd.Series):
+ assert len(self.target_names) == 1, "Not enough columns in y_pred"
+ y_pred = pd.DataFrame({self.target_names[0]: y_pred})
+ # TODO auto-create the timestamps for the time column instead of throwing
+ raise NotImplementedError("Need a non-None test_data for this to work, for now")
+
+ assert isinstance(y_pred, pd.DataFrame)
+ assert self.time_col in y_pred.columns
+ assert all([t in y_pred.columns for t in self.target_names])
+ return y_pred
+
+ def merge_prediction_with_target(self, y_pred: Union[pd.DataFrame, pd.Series, np.ndarray]):
+ y_pred = self.prettify_prediction(y_pred)
+ return pd.concat([self.train_data[[self.time_col] + self.target_names], y_pred], axis=0)
+
+
+def enrich_dataframe(
+ df: Union[pd.DataFrame, pd.Series],
+ fourier_degree: int,
+ remove_constants: bool = False,
+ fourier_time: bool = True,
+) -> pd.DataFrame:
+ if isinstance(df, pd.Series):
+ df = pd.DataFrame(df)
+
+ new_cols = []
+ for col in df.columns:
+ if df[col].dtype.name == "datetime64[ns]":
+ extras = monthly_fourier_features(df[col], fourier_degree)
+ extras.columns = [f"{col}_{c}" for c in extras.columns]
+ extras.index = df.index
+ new_cols.append(extras)
+ date_feat = date_feature_dict_fourier(df[col]) if fourier_time else date_feature_dict(df[col])
+ if remove_constants:
+ re_date_feat = {k: v for k, v in date_feat.items() if v.nunique(dropna=False) >= 2}
+ else:
+ re_date_feat = date_feat
+
+ date_feat = pd.DataFrame(re_date_feat, index=df.index)
+ new_cols.append(date_feat)
+
+ return pd.concat([df] + new_cols, axis=1, verify_integrity=True)
+
+
+def enrich_dataset(
+ X: TimeSeriesDataset,
+ fourier_degree: int = 0,
+ remove_constants: bool = False,
+ fourier_time: bool = True,
+) -> TimeSeriesDataset:
+ new_train = enrich_dataframe(X.train_data, fourier_degree, remove_constants, fourier_time)
+ new_test = (
+ None if X.test_data is None else enrich_dataframe(X.test_data, fourier_degree, remove_constants, fourier_time)
+ )
+ return TimeSeriesDataset(
+ train_data=new_train,
+ time_col=X.time_col,
+ target_names=X.target_names,
+ time_idx=X.time_idx,
+ test_data=new_test,
+ )
+
+
+def date_feature_dict(timestamps: pd.Series) -> dict:
+ tmp_dt = timestamps.dt
+ column = timestamps.name
+ pre_columns_dict = {
+ # f"{column}_year": tmp_dt.year, # not stationary
+ f"{column}_month": tmp_dt.month,
+ # f"{column}_day": tmp_dt.day,# taken care of with monthly fourier features
+ f"{column}_hour": tmp_dt.hour,
+ f"{column}_minute": tmp_dt.minute,
+ f"{column}_second": tmp_dt.second,
+ f"{column}_dayofweek": tmp_dt.dayofweek,
+ f"{column}_dayofyear": tmp_dt.dayofyear,
+ f"{column}_quarter": tmp_dt.quarter,
+ }
+
+ new_columns_dict = {}
+ for k, v in pre_columns_dict.items():
+ new_columns_dict.update(fourier_series(v, k))
+
+ return new_columns_dict
+
+
+def date_feature_dict_fourier(timestamps: pd.Series) -> dict:
+ tmp_dt = timestamps.dt
+ column = timestamps.name
+ pre_columns_dict = {
+ # f"{column}_year": tmp_dt.year, # not stationary
+ f"{column}_month": tmp_dt.month / 12.0,
+ # f"{column}_day": tmp_dt.day,# taken care of with monthly fourier features
+ f"{column}_hour": tmp_dt.hour / 24.0,
+ f"{column}_minute": tmp_dt.minute / 60.0,
+ f"{column}_second": tmp_dt.second / 60.0,
+ f"{column}_dayofweek": tmp_dt.dayofweek / 7.0,
+ f"{column}_dayofyear": tmp_dt.dayofyear / 366.0,
+ f"{column}_quarter": tmp_dt.quarter / 4.0,
+ }
+
+ new_columns_dict = {}
+ for k, v in pre_columns_dict.items():
+ new_columns_dict.update(fourier_series(v, k))
+
+ return new_columns_dict
+
+
+def fourier_series(feature: pd.Series, name: str):
+ """
+ Assume feature goes from 0 to 1 cyclically, transform that into Fourier
+ @param feature: input feature
+ @return: sin(2pi*feature), cos(2pi*feature)
+ """
+ return {
+ name + "_sin": np.sin(2 * math.pi * feature),
+ name + "_cos": np.cos(2 * math.pi * feature),
+ }
+
+
+class DataTransformerTS:
+ """Transform input time series training data."""
+
+ def __init__(self, time_col: str, label: Union[str, List[str]], time_idx: str = "time_idx"):
+ self.time_col = time_col
+ self.time_idx = time_idx
+ self.label = label
+ self.cat_columns = []
+ self.num_columns = []
+ self.datetime_columns = []
+ self.drop_columns = []
+
+ @property
+ def _drop(self):
+ return len(self.drop_columns)
+
+ def fit(self, X: Union[DataFrame, np.array], y):
+ """Fit transformer.
+
+ Args:
+ X: A numpy array or a pandas dataframe of training data.
+ y: A numpy array or a pandas series of labels.
+
+ Returns:
+ X: Processed numpy array or pandas dataframe of training data.
+ y: Processed numpy array or pandas series of labels.
+ """
+ assert isinstance(X, DataFrame)
+ X = X.copy()
+ n = X.shape[0]
+
+ assert len(self.num_columns) == 0, "Trying to call fit() twice, something is wrong"
+
+ for column in X.columns:
+ # sklearn/utils/validation.py needs int/float values
+ if X[column].dtype.name in ("object", "category"):
+ if (
+ # drop columns where all values are the same
+ X[column].nunique() == 1
+ # this drops UID-type cols
+ or X[column].nunique(dropna=True) == n - X[column].isnull().sum()
+ ):
+ self.drop_columns.append(column)
+ elif column != self.time_idx:
+ self.cat_columns.append(column)
+ elif X[column].nunique(dropna=True) < 2:
+ self.drop_columns.append(column)
+ elif X[column].dtype.name == "datetime64[ns]":
+ pass # these will be processed at model level,
+ # so they can also be done in the predict method
+ else:
+ self.num_columns.append(column)
+
+ if self.num_columns:
+ self.transformer = ColumnTransformer(
+ [
+ (
+ "continuous",
+ SimpleImputer(missing_values=np.nan, strategy="median"),
+ self.num_columns,
+ )
+ ]
+ )
+
+ self.transformer.fit(X[self.num_columns])
+ else:
+ self.transformer = None
+
+ # TODO: revisit for multivariate series, and recast for a single df input anyway
+ if isinstance(y, Series):
+ y = y.rename(self.label)
+
+ if isinstance(y, pd.DataFrame):
+ ycol = y[y.columns[0]]
+ elif isinstance(y, pd.Series):
+ ycol = y
+ else:
+ raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")
+
+ if not pd.api.types.is_numeric_dtype(ycol):
+ self.label_transformer = LabelEncoder()
+ self.label_transformer.fit(ycol)
+ else:
+ self.label_transformer = None
+
+ def transform(self, X: Union[DataFrame, np.array], y=None):
+ # TODO: revisit for multivariate series, and recast for a single df input anyway
+ if self.label_transformer is not None and y is not None:
+ if isinstance(y, pd.DataFrame):
+ ycol = y[y.columns[0]]
+ elif isinstance(y, pd.Series):
+ ycol = y
+ else:
+ raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")
+ y_tr = self.label_transformer.transform(ycol)
+ y.iloc[:] = y_tr.reshape(y.shape)
+
+ X.drop(columns=self.drop_columns, inplace=True)
+
+ for col in self.cat_columns:
+ if X[col].dtype.name == "category":
+ if "__NAN__" not in X[col].cat.categories:
+ X[col] = X[col].cat.add_categories("__NAN__").fillna("__NAN__")
+ else:
+ X[col] = X[col].fillna("__NAN__")
+ X[col] = X[col].astype("category")
+
+ for column in self.num_columns:
+ X[column] = X[column].fillna(np.nan)
+
+ if self.transformer is not None:
+ X[self.num_columns] = self.transformer.transform(X[self.num_columns])
+
+ if y is None:
+ return X
+ return X, y
+
+ def fit_transform(self, X: Union[DataFrame, np.array], y):
+ self.fit(X, y)
+ return self.transform(X, y)
+
+
+def create_forward_frame(
+ frequency: str,
+ steps: int,
+ test_end_date: datetime.datetime,
+ time_col: str,
+):
+ start_date = test_end_date + pd.Timedelta(1, frequency)
+ times = pd.date_range(
+ start=start_date,
+ periods=steps,
+ freq=frequency,
+ )
+ return pd.DataFrame({time_col: times})
+
+
+def normalize_ts_data(X_train_all, target_names, time_col, y_train_all=None):
+ if isinstance(X_train_all, TimeSeriesDataset):
+ return X_train_all
+
+ if issparse(X_train_all):
+ X_train_all = X_train_all.tocsr()
+
+ if isinstance(X_train_all, np.ndarray) and len(X_train_all.shape) == 1:
+ X_train_all = np.reshape(X_train_all, (X_train_all.size, 1))
+
+ if isinstance(X_train_all, np.ndarray):
+ X_train_all = pd.DataFrame(
+ X_train_all,
+ columns=[time_col] + [f"x{i}" for i in range(X_train_all.shape[1] - 1)],
+ )
+
+ if y_train_all is None:
+ return X_train_all
+ else:
+ if isinstance(y_train_all, np.ndarray):
+ # TODO: will need to revisit this when doing multivariate y
+ y_train_all = pd.DataFrame(
+ y_train_all.reshape(len(X_train_all), -1),
+ columns=target_names,
+ index=X_train_all.index,
+ )
+ elif isinstance(y_train_all, pd.Series):
+ y_train_all = pd.DataFrame(y_train_all)
+ y_train_all.index = X_train_all.index
+
+ dataframe = pd.concat([X_train_all, y_train_all], axis=1)
+
+ return dataframe
+
+
+def validate_data_basic(X_train_all, y_train_all):
+ assert isinstance(X_train_all, np.ndarray) or issparse(X_train_all) or isinstance(X_train_all, pd.DataFrame), (
+ "X_train_all must be a numpy array, a pandas dataframe, " "or Scipy sparse matrix."
+ )
+
+ assert (
+ isinstance(y_train_all, np.ndarray)
+ or isinstance(y_train_all, pd.Series)
+ or isinstance(y_train_all, pd.DataFrame)
+ ), "y_train_all must be a numpy array or a pandas series or DataFrame."
+
+ assert X_train_all.size != 0 and y_train_all.size != 0, "Input data must not be empty, use None if no data"
+
+ assert X_train_all.shape[0] == y_train_all.shape[0], "# rows in X_train must match length of y_train."
diff --git a/flaml/automl/time_series/ts_model.py b/flaml/automl/time_series/ts_model.py
new file mode 100644
index 000000000..da1bfcbaf
--- /dev/null
+++ b/flaml/automl/time_series/ts_model.py
@@ -0,0 +1,760 @@
+import time
+import logging
+import os
+from datetime import datetime
+import math
+from typing import List, Optional, Union
+
+try:
+ import pandas as pd
+ from pandas import DataFrame, Series, to_datetime
+except ImportError:
+
+ class PD:
+ pass
+
+ pd = PD()
+ pd.DataFrame = None
+ pd.Series = None
+ DataFrame = Series = None
+
+
+import numpy as np
+
+from flaml import tune
+from flaml.model import (
+ suppress_stdout_stderr,
+ SKLearnEstimator,
+ logger,
+ LGBMEstimator,
+ XGBoostSklearnEstimator,
+ RandomForestEstimator,
+ ExtraTreesEstimator,
+ XGBoostLimitDepthEstimator,
+ CatBoostEstimator,
+)
+from flaml.data import TS_TIMESTAMP_COL, TS_VALUE_COL
+from flaml.automl.time_series.ts_data import (
+ TimeSeriesDataset,
+ enrich_dataset,
+ enrich_dataframe,
+ normalize_ts_data,
+ create_forward_frame,
+)
+from flaml.automl.task import Task
+
+
+class TimeSeriesEstimator(SKLearnEstimator):
+ def __init__(self, task="ts_forecast", n_jobs=1, **params):
+ super().__init__(task, **params)
+ self.time_col: Optional[str] = None
+ self.target_names: Optional[Union[str, List[str]]] = None
+ self.frequency: Optional[str] = None
+ self.end_date: Optional[datetime] = None
+ self.regressors: Optional[List[str]] = None
+
+ def enrich(
+ self,
+ X: Union[int, TimeSeriesDataset, DataFrame],
+ remove_constants: bool = False,
+ ):
+ X = normalize_ts_data(X, None, self.time_col, None)
+ if isinstance(X, int):
+ X = create_forward_frame(self.frequency, X, self.end_date, self.time_col)
+
+ fourier_degree = self.params.get("monthly_fourier_degree", 4)
+
+ if isinstance(X, TimeSeriesDataset):
+ return enrich_dataset(
+ X,
+ fourier_degree,
+ remove_constants=remove_constants,
+ fourier_time=self.params.get("fourier_time_features"),
+ )
+
+ return enrich_dataframe(
+ X,
+ fourier_degree,
+ remove_constants=remove_constants,
+ fourier_time=self.params.get("fourier_time_features"),
+ )
+
+ @classmethod
+ def search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int):
+ space = cls._search_space(data=data, task=task, pred_horizon=pred_horizon)
+ space.update(cls.top_search_space())
+ return space
+
+ @staticmethod
+ def adjust_scale(scale: int, data_len: int, pred_horizon: int):
+ points = data_len - pred_horizon
+ max_lags = math.floor(points / scale)
+
+ while scale > 2:
+ if max_lags >= 2:
+ break
+ scale = math.ceil(scale / 1.7)
+ max_lags = math.floor(points / scale)
+
+ assert scale >= 2 and max_lags >= 2, f"Too few points ({data_len}) for prediction horizon {pred_horizon}"
+
+ return scale, max_lags
+
+ @classmethod
+ def top_search_space(cls):
+ return {
+ "monthly_fourier_degree": {
+ "domain": tune.randint(lower=0, upper=8),
+ "init_value": 4,
+ "low_cost_init_value": 2,
+ },
+ "fourier_time_features": {
+ "domain": tune.randint(lower=0, upper=2), # tune.choice([True, False]),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "pca_features": { # disable for now, will deal with occasional svd fail later
+ "domain": tune.choice([False]),
+ "init_value": False,
+ "low_cost_init_value": False,
+ },
+ }
+
+ @classmethod
+ def top_level_params(cls):
+ return ["monthly_fourier_degree"]
+
+ def _join(self, X_train, y_train):
+ assert TS_TIMESTAMP_COL in X_train, (
+ "Dataframe for training ts_forecast model must have column"
+ f' "{TS_TIMESTAMP_COL}" with the dates in X_train.'
+ )
+ y_train = DataFrame(y_train, columns=[TS_VALUE_COL])
+ train_df = X_train.join(y_train)
+ return train_df
+
+ def fit(self, X_train: TimeSeriesDataset, y_train=None, budget=None, **kwargs):
+ # TODO purge y_train
+ self.time_col = X_train.time_col
+ self.target_names = X_train.target_names
+ self.X_train = X_train
+ self.frequency = self.X_train.frequency
+ self.end_date = self.X_train.end_date
+
+ def score(self, X_val: DataFrame, y_val: Series, **kwargs):
+ from sklearn.metrics import r2_score
+ from ..ml import metric_loss_score
+
+ y_pred = self.predict(X_val, **kwargs)
+ if isinstance(X_val, TimeSeriesDataset):
+ y_val = X_val.test_data[X_val.target_names[0]]
+ self._metric = kwargs.get("metric", None)
+ if self._metric:
+ return metric_loss_score(self._metric, y_pred, y_val)
+ else:
+ return r2_score(y_pred, y_val)
+
+
+class Orbit(TimeSeriesEstimator):
+ def fit(self, X_train: TimeSeriesDataset, y_train=None, budget=None, **kwargs):
+ # This may be needed to get PyStan to run, needed for Orbit
+ os.environ["KMP_DUPLICATE_LIB_OK"] = "True"
+ from orbit.models import DLT
+
+ # y_train is ignored, just need it for signature compatibility with other classes
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+ current_time = time.time()
+ self.logger = logging.getLogger("orbit").setLevel(logging.WARNING)
+
+ model_class = self.params.get("model_class", DLT)
+ self._model = model_class(
+ response_col=X_train.target_names[0],
+ date_col=X_train.time_col,
+ regressor_col=X_train.regressors,
+ # TODO: infer seasonality from frequency
+ **self.params,
+ )
+
+ with suppress_stdout_stderr():
+ self._model.fit(df=X_train.train_data.copy())
+
+ train_time = time.time() - current_time
+ return train_time
+
+ def predict(self, X: Union[TimeSeriesDataset, DataFrame], **kwargs):
+ if isinstance(X, int):
+ X = create_forward_frame(
+ self.frequency,
+ X,
+ self.end_date,
+ self.time_col,
+ )
+
+ elif isinstance(X, TimeSeriesDataset):
+ data = X
+ X = data.test_data[[self.time_col] + X.regressors]
+
+ if self._model is not None:
+ forecast = self._model.predict(X, **kwargs)
+ out = (
+ DataFrame(
+ forecast[
+ [
+ self.time_col,
+ "prediction",
+ "prediction_5",
+ "prediction_95",
+ ]
+ ]
+ )
+ .reset_index(drop=True)
+ .rename(
+ columns={
+ "prediction": self.target_names[0],
+ }
+ )
+ )
+
+ return out
+ else:
+ self.logger.warning("Estimator is not fit yet. Please run fit() before predict().")
+ return None
+
+ @classmethod
+ def _search_space(cls, **params):
+ # TODO: fill in a proper search space
+ space = {}
+ return space
+
+
+class Prophet(TimeSeriesEstimator):
+ """The class for tuning Prophet."""
+
+ @classmethod
+ def _search_space(cls, **params):
+ space = {
+ "changepoint_prior_scale": {
+ "domain": tune.loguniform(lower=0.001, upper=0.05),
+ "init_value": 0.05,
+ "low_cost_init_value": 0.001,
+ },
+ "seasonality_prior_scale": {
+ "domain": tune.loguniform(lower=0.01, upper=10),
+ "init_value": 10,
+ },
+ "holidays_prior_scale": {
+ "domain": tune.loguniform(lower=0.01, upper=10),
+ "init_value": 10,
+ },
+ "seasonality_mode": {
+ "domain": tune.choice(["additive", "multiplicative"]),
+ "init_value": "multiplicative",
+ },
+ }
+ return space
+
+ def fit(self, X_train, y_train=None, budget=None, **kwargs):
+ from prophet import Prophet
+
+ X_train = self.enrich(X_train)
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+
+ current_time = time.time()
+
+ if isinstance(X_train, TimeSeriesDataset):
+ data = X_train
+ target_col = data.target_names[0]
+ time_col = data.time_col
+ regressors = data.regressors
+ # this class only supports univariate regression
+ train_df = data.train_data[regressors + [target_col, time_col]]
+ train_df = train_df.rename(columns={target_col: "y", time_col: "ds"})
+ else:
+ train_df = self._join(X_train, y_train)
+
+ regressors = list(train_df.columns)
+ regressors.remove(TS_TIMESTAMP_COL)
+ regressors.remove(TS_VALUE_COL)
+
+ train_df = self._preprocess(train_df)
+ logging.getLogger("prophet").setLevel(logging.WARNING)
+ nice_params = {k: v for k, v in self.params.items() if k in self._search_space()}
+ model = Prophet(**nice_params)
+ for regressor in regressors:
+ model.add_regressor(regressor)
+ with suppress_stdout_stderr():
+ model.fit(train_df)
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+ def predict(self, X, **kwargs):
+ X = self.enrich(X)
+ if isinstance(X, int):
+ raise ValueError(
+ "predict() with steps is only supported for arima/sarimax."
+ " For Prophet, pass a dataframe with the first column containing"
+ " the timestamp values."
+ )
+
+ if isinstance(X, TimeSeriesDataset):
+ data = X
+ X = data.test_data[data.regressors + [data.time_col]]
+
+ X = X.rename(columns={self.time_col: "ds"})
+ if self._model is not None:
+ X = self._preprocess(X)
+ forecast = self._model.predict(X, **kwargs)
+ out = forecast["yhat"]
+ out.name = self.target_names[0]
+ return out
+
+ else:
+ logger.warning("Estimator is not fit yet. Please run fit() before predict().")
+ return np.ones(X.shape[0])
+
+
+class StatsModelsEstimator(TimeSeriesEstimator):
+ def predict(self, X, **kwargs) -> pd.Series:
+ X = self.enrich(X)
+ if self._model is None or self._model is False:
+ return np.ones(X if isinstance(X, int) else X.shape[0])
+
+ if isinstance(X, int):
+ return self._model.forecast(steps=X)
+
+ if isinstance(X, TimeSeriesDataset):
+ data = X
+ X = data.test_data[data.regressors + [data.time_col]]
+ else:
+ X = X[self.regressors + [self.time_col]]
+
+ if isinstance(X, DataFrame):
+ start = X[self.time_col].iloc[0]
+ end = X[self.time_col].iloc[-1]
+ if len(self.regressors):
+ exog = self._preprocess(X[self.regressors])
+ forecast = self._model.predict(start=start, end=end, exog=exog.values, **kwargs)
+ else:
+ forecast = self._model.predict(start=start, end=end, **kwargs)
+ else:
+ raise ValueError(
+ "X needs to be either a pandas Dataframe with dates as the first column"
+ " or an int number of periods for predict()."
+ )
+ forecast.name = self.target_names[0]
+ return forecast
+
+
+class ARIMA(StatsModelsEstimator):
+ """The class for tuning ARIMA."""
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ if not all([p in self.params for p in ["p", "d", "q"]]):
+ print("arima params at init time:")
+ print(self.params)
+ try:
+ raise ValueError("ARIMA initialized without required params p, d, q")
+ except Exception as e:
+ import traceback
+
+ print(traceback.format_exc())
+ raise e
+
+ @classmethod
+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
+ scale, _ = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)
+ space = {
+ "p": {
+ "domain": tune.qrandint(lower=0, upper=2 * scale, q=1),
+ "init_value": scale,
+ "low_cost_init_value": 0,
+ },
+ "d": {
+ "domain": tune.qrandint(lower=0, upper=6, q=1),
+ "init_value": 1,
+ "low_cost_init_value": 0,
+ },
+ "q": {
+ "domain": tune.qrandint(lower=0, upper=2 * scale, q=1),
+ "init_value": scale,
+ "low_cost_init_value": 0,
+ },
+ }
+ return space
+
+ def _join(self, X_train, y_train):
+ train_df = super()._join(X_train, y_train)
+ train_df.index = to_datetime(train_df[TS_TIMESTAMP_COL])
+ train_df = train_df.drop(TS_TIMESTAMP_COL, axis=1)
+ return train_df
+
+ def fit(self, X_train, y_train=None, budget=None, **kwargs):
+ import warnings
+
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+ X_train = self.enrich(X_train, remove_constants=True)
+
+ warnings.filterwarnings("ignore")
+ from statsmodels.tsa.arima.model import ARIMA as ARIMA_estimator
+
+ current_time = time.time()
+
+ if isinstance(X_train, TimeSeriesDataset):
+ data = X_train
+ # this class only supports univariate regression
+ target_col = data.target_names[0] if isinstance(data.target_names, list) else data.target_names
+ self.regressors = data.regressors
+ train_df = data.train_data[self.regressors + [target_col]]
+ train_df.index = to_datetime(data.train_data[data.time_col])
+ self.time_col = data.time_col
+ self.target_names = target_col
+ else:
+ target_col = TS_VALUE_COL
+ train_df = self._join(X_train, y_train)
+ self.regressors = list(train_df)
+ self.regressors.remove(TS_VALUE_COL)
+
+ train_df = self._preprocess(train_df)
+
+ if len(self.regressors):
+ model = ARIMA_estimator(
+ train_df[[target_col]],
+ exog=train_df[self.regressors],
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ else:
+ model = ARIMA_estimator(
+ train_df,
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ with suppress_stdout_stderr():
+ model = model.fit()
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+
+class SARIMAX(StatsModelsEstimator):
+ """The class for tuning SARIMA."""
+
+ @classmethod
+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
+ scale, max_lags = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)
+
+ # TODO: instead, downscale the dataset and take next_scale from that for P and Q
+ scales = [
+ s for s in [scale, 2 * scale, 3 * scale, 4 * scale] if s * max_lags <= len(data.train_data) - pred_horizon
+ ]
+
+ space = {
+ "p": {
+ "domain": tune.qrandint(lower=0, upper=scale - 1, q=1),
+ "init_value": scale - 1,
+ "low_cost_init_value": 0,
+ },
+ "d": {
+ "domain": tune.qrandint(lower=0, upper=6, q=1),
+ "init_value": 0,
+ "low_cost_init_value": 0,
+ },
+ "q": {
+ "domain": tune.qrandint(lower=0, upper=scale - 1, q=1),
+ "init_value": scale - 1,
+ "low_cost_init_value": 0,
+ },
+ "P": {
+ "domain": tune.qrandint(lower=0, upper=min(10, max_lags), q=1),
+ "init_value": 3,
+ "low_cost_init_value": 0,
+ },
+ "D": {
+ "domain": tune.qrandint(lower=0, upper=6, q=1),
+ "init_value": 0,
+ "low_cost_init_value": 0,
+ },
+ "Q": {
+ "domain": tune.qrandint(lower=0, upper=min(10, max_lags), q=1),
+ "init_value": 3,
+ "low_cost_init_value": 0,
+ },
+ "s": {
+ "domain": tune.choice(scales),
+ "init_value": scale,
+ },
+ }
+ return space
+
+ def fit(self, X_train, y_train=None, budget=None, **kwargs):
+ import warnings
+
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+ X_train = self.enrich(X_train)
+
+ warnings.filterwarnings("ignore")
+ from statsmodels.tsa.statespace.sarimax import SARIMAX as SARIMAX_estimator
+
+ current_time = time.time()
+
+ if isinstance(X_train, TimeSeriesDataset):
+ data = X_train
+ target_col = data.target_names[0]
+ self.regressors = data.regressors
+ # this class only supports univariate regression
+ train_df = data.train_data[self.regressors + [target_col]]
+ train_df.index = to_datetime(data.train_data[data.time_col])
+ else:
+ target_col = TS_VALUE_COL
+ train_df = self._join(X_train, y_train)
+ self.regressors = list(train_df)
+ self.regressors.remove(TS_VALUE_COL)
+
+ train_df = self._preprocess(train_df)
+ # regressors = list(train_df)
+ # regressors.remove(target_col)
+ if self.regressors:
+ model = SARIMAX_estimator(
+ train_df[[target_col]],
+ exog=train_df[self.regressors],
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ seasonal_order=(
+ self.params["P"],
+ self.params["D"],
+ self.params["Q"],
+ self.params["s"],
+ ),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ else:
+ model = SARIMAX_estimator(
+ train_df,
+ order=(self.params["p"], self.params["d"], self.params["q"]),
+ seasonal_order=(
+ self.params["P"],
+ self.params["D"],
+ self.params["Q"],
+ self.params["s"],
+ ),
+ enforce_stationarity=False,
+ enforce_invertibility=False,
+ )
+ with suppress_stdout_stderr():
+ model = model.fit()
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+
+class HoltWinters(StatsModelsEstimator):
+ """
+ The class for tuning Holt Winters model, aka 'Triple Exponential Smoothing'.
+ """
+
+ @classmethod
+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
+ space = {
+ "damped_trend": {"domain": tune.choice([True, False]), "init_value": False},
+ "trend": {"domain": tune.choice(["add", "mul", None]), "init_value": "add"},
+ "seasonal": {
+ "domain": tune.choice(["add", "mul", None]),
+ "init_value": "add",
+ },
+ "use_boxcox": {"domain": tune.choice([False, True]), "init_value": False},
+ "seasonal_periods": { # statsmodels casts this to None if "seasonal" is None
+ "domain": tune.choice([7, 12, 4, 52, 6]), # weekly, yearly, quarterly, weekly w yearly data
+ "init_value": 7,
+ },
+ }
+ return space
+
+ def fit(self, X_train, y_train, budget=None, free_mem_ratio=0, **kwargs):
+ import warnings
+
+ warnings.filterwarnings("ignore")
+ from statsmodels.tsa.holtwinters import (
+ ExponentialSmoothing as HWExponentialSmoothing,
+ )
+
+ current_time = time.time()
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+ X_train = self.enrich(X_train)
+
+ self.regressors = []
+ if isinstance(X_train, TimeSeriesDataset):
+ data = X_train
+ target_col = data.target_names[0]
+ regressors = data.regressors
+ # this class only supports univariate regression
+ train_df = data.train_data[self.regressors + [target_col]]
+ train_df.index = to_datetime(data.train_data[data.time_col])
+ else:
+ target_col = TS_VALUE_COL
+ train_df = self._join(X_train, y_train)
+ regressors = list(train_df)
+ regressors.remove(TS_VALUE_COL)
+
+ if regressors:
+ logger.warning("Regressors are ignored for Holt-Winters ETS models.")
+
+ train_df = self._preprocess(train_df)
+
+ # Override incompatible parameters
+ if (
+ train_df.shape[0] < 2 * self.params["seasonal_periods"]
+ ): # this would prevent heuristic initialization to work properly
+ self.params["seasonal"] = None
+ if (
+ self.params["seasonal"] == "mul" and (train_df.y == 0).sum() > 0
+ ): # cannot have multiplicative seasonality in this case
+ self.params["seasonal"] = "add"
+ if self.params["trend"] == "mul" and (train_df.y == 0).sum() > 0:
+ self.params["trend"] = "add"
+
+ if not self.params["seasonal"] or self.params["trend"] not in ["mul", "add"]:
+ self.params["damped_trend"] = False
+
+ model = HWExponentialSmoothing(
+ train_df[[target_col]],
+ damped_trend=self.params["damped_trend"],
+ seasonal=self.params["seasonal"],
+ trend=self.params["trend"],
+ )
+ with suppress_stdout_stderr():
+ model = model.fit()
+ train_time = time.time() - current_time
+ self._model = model
+ return train_time
+
+
+class TS_SKLearn(TimeSeriesEstimator):
+ """The class for tuning SKLearn Regressors for time-series forecasting"""
+
+ base_class = SKLearnEstimator
+
+ @classmethod
+ def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
+ data_size = data.train_data.shape
+ space = cls.base_class.search_space(data_size=data_size, task=task, **params)
+
+ scale, _ = cls.adjust_scale(data.next_scale(), len(data.train_data), pred_horizon)
+
+ max_lags = max(3 * scale, int(np.sqrt(data_size[0])))
+ max_lags = min(max_lags, data_size[0] - pred_horizon - 1)
+
+ space.update(
+ {
+ "lags": {
+ "domain": tune.randint(lower=1, upper=max_lags),
+ "init_value": min(max_lags, scale),
+ },
+ }
+ )
+ return space
+
+ def __init__(self, task="ts_forecast", **params):
+ # TODO: pass task objects throughout
+ super().__init__(task, **params)
+ self._model = None
+ self.ts_task = task
+
+ def fit(self, X_train, y_train=None, budget=None, **kwargs):
+ super().fit(X_train, y_train, budget=budget, **kwargs)
+ X_train = self.enrich(X_train)
+
+ current_time = time.time()
+ if isinstance(X_train, TimeSeriesDataset):
+ data = X_train
+ X_train = data.train_data[data.regressors + [data.time_col]]
+ self.regressors = data.regressors
+ # this class only supports univariate regression
+ y_train = data.y_train
+ self.time_col = data.time_col
+ self.target_names = data.target_names
+ elif isinstance(X_train, DataFrame):
+ self.time_col = X_train.columns.tolist()[0]
+
+ # X_train = self.transform_X(X_train)
+ self.regressors = X_train.columns.tolist()[1:]
+ else:
+ raise ValueError("Unknown X type")
+
+ X_train = self._preprocess(X_train)
+
+ est_params = {k: v for k, v in self.params.items() if k not in self.top_search_space().keys()}
+
+ from flaml.automl.time_series.sklearn import SklearnWrapper
+
+ horizon = kwargs.pop("period")
+ lags = est_params.pop("lags")
+ est_params["task"] = self._task
+ self._model = SklearnWrapper(
+ self.base_class,
+ horizon=horizon,
+ lags=lags,
+ init_params=est_params,
+ pca_features=self.params.get("pca_features", False),
+ )
+ self._model.fit(X_train[self.regressors], y_train)
+
+ train_time = time.time() - current_time
+ return train_time
+
+ def predict(self, X, **kwargs):
+ X = self.enrich(X)
+ if isinstance(X, TimeSeriesDataset):
+ data = X
+ X = data.test_data
+
+ if self._model is not None:
+ X = X[self.regressors]
+ # X = self.transform_X(X)
+ X = self._preprocess(X)
+ forecast = self._model.predict(X)
+ if isinstance(forecast, Series):
+ forecast.name = self.target_names[0]
+
+ return forecast
+ else:
+ logger.warning("Estimator is not fit yet. Please run fit() before predict().")
+ return np.ones(X.shape[0])
+
+
+class LGBM_TS(TS_SKLearn):
+ """The class for tuning LGBM Regressor for time-series forecasting"""
+
+ base_class = LGBMEstimator
+
+
+class XGBoost_TS(TS_SKLearn):
+ """The class for tuning XGBoost Regressor for time-series forecasting"""
+
+ base_class = XGBoostSklearnEstimator
+
+
+class RF_TS(TS_SKLearn):
+ """The class for tuning Random Forest Regressor for time-series forecasting"""
+
+ base_class = RandomForestEstimator
+
+
+class ExtraTrees_TS(TS_SKLearn):
+ """The class for tuning Extra Trees Regressor for time-series forecasting"""
+
+ base_class = ExtraTreesEstimator
+
+
+class XGBoostLimitDepth_TS(TS_SKLearn):
+ """The class for tuning XGBoost Regressor with unlimited depth for time-series forecasting"""
+
+ base_class = XGBoostLimitDepthEstimator
+
+
+# catboost regressor is invalid because it has a `name` parameter, making it incompatible with hcrystalball
+class CatBoost_TS(TS_SKLearn):
+ base_class = CatBoostEstimator
diff --git a/flaml/automl/training_log.py b/flaml/automl/training_log.py
index c49c6e592..0c01c3f6a 100644
--- a/flaml/automl/training_log.py
+++ b/flaml/automl/training_log.py
@@ -29,7 +29,7 @@ class TrainingLogRecord(object):
self.logged_metric = logged_metric
self.trial_time = trial_time
self.wall_clock_time = wall_clock_time
- self.validation_loss = validation_loss
+ self.validation_loss = float(validation_loss)
self.config = config
self.learner = learner
self.sample_size = sample_size
diff --git a/flaml/default/suggest.py b/flaml/default/suggest.py
index 12acc3678..05ff342eb 100644
--- a/flaml/default/suggest.py
+++ b/flaml/default/suggest.py
@@ -5,7 +5,7 @@ import json
from flaml.automl.data import DataTransformer
from flaml.automl.task.task import CLASSIFICATION, get_classification_objective
from flaml.automl.task.generic_task import len_labels
-from flaml.automl.ml import get_estimator_class
+from flaml.automl.task.factory import task_factory
from flaml.version import __version__
try:
@@ -162,9 +162,10 @@ def suggest_hyperparams(task, X, y, estimator_or_predictor, location=None):
"""
config = suggest_config(task, X, y, estimator_or_predictor, location=location, k=1)[0]
estimator = config["class"]
- model_class = get_estimator_class(task, estimator)
+ task = task_factory(task)
+ model_class = task.estimator_class_from_str(estimator)
hyperparams = config["hyperparameters"]
- model = model_class(task=task, **hyperparams)
+ model = model_class(task=task.name, **hyperparams)
estimator_class = model.estimator_class
hyperparams = hyperparams and model.params
return hyperparams, estimator_class
@@ -246,7 +247,7 @@ def preprocess_and_suggest_hyperparams(
)
config = suggest_config(task, X, y, estimator_or_predictor, location=location, k=1)[0]
estimator = config["class"]
- model_class = get_estimator_class(task, estimator)
+ model_class = task_factory(task).estimator_class_from_str(estimator)
hyperparams = config["hyperparameters"]
model = model_class(task=task, **hyperparams)
if model.estimator_class is None:
diff --git a/setup.py b/setup.py
index c29272b38..fd588084d 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,7 @@ setuptools.setup(
"nbformat",
"ipykernel",
"pytorch-lightning<1.9.1", # test_forecast_panel
+ "tensorboardX==2.6", # test_forecast_panel
"requests<2.29.0", # https://github.com/docker/docker-py/issues/3113
"packaging",
],
@@ -125,6 +126,8 @@ setuptools.setup(
"statsmodels>=0.12.2",
"hcrystalball==0.1.10",
"pytorch-forecasting>=0.9.0",
+ "pytorch-lightning==1.9.0",
+ "tensorboardX==2.6",
],
"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],
"openai": ["openai==0.27.8", "diskcache"],
diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.py
index 044d11a72..19997c3c8 100644
--- a/test/automl/test_forecast.py
+++ b/test/automl/test_forecast.py
@@ -1,8 +1,14 @@
+import datetime
+
import numpy as np
+import pandas as pd
+
from flaml import AutoML
+from flaml.automl.task.time_series_task import TimeSeriesTask
-def test_forecast_automl(budget=5, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]):
+
+def test_forecast_automl(budget=10, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]):
# using dataframe
import statsmodels.api as sm
@@ -88,6 +94,32 @@ def test_forecast_automl(budget=5, estimators_when_no_prophet=["arima", "sarimax
)
+def test_models(budget=3):
+ n = 100
+ X = pd.DataFrame(
+ {
+ "A": pd.date_range(start="1900-01-01", periods=n, freq="D"),
+ }
+ )
+ y = np.exp(np.random.randn(n))
+
+ task = TimeSeriesTask("ts_forecast")
+
+ for est in task.estimators.keys():
+ if est == "tft":
+ continue # TFT is covered by its own test
+ automl = AutoML()
+ automl.fit(
+ X_train=X[:72], # a single column of timestamp
+ y_train=y[:72], # value for each timestamp
+ estimator_list=[est],
+ period=12, # time horizon to forecast, e.g., 12 months
+ task="ts_forecast",
+ time_budget=budget, # time budget in seconds
+ )
+ automl.predict(X[72:])
+
+
def test_numpy():
X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]")
y_train = np.random.random(size=len(X_train))
@@ -500,7 +532,7 @@ def test_forecast_panel(budget=5):
],
"time_varying_unknown_categoricals": [],
"time_varying_unknown_reals": [
- "y", # always need a 'y' column for the target column
+ "volume", # target column
"log_volume",
"industry_volume",
"soda_volume",
@@ -572,10 +604,69 @@ def test_forecast_panel(budget=5):
print(automl.min_resource)
+def test_cv_step():
+ n = 300
+ time_col = "date"
+ df = pd.DataFrame(
+ {
+ time_col: pd.date_range(start="1/1/2001", periods=n, freq="D"),
+ "y": np.sin(np.linspace(start=0, stop=200, num=n)),
+ }
+ )
+
+ def split_by_date(df: pd.DataFrame, dt: datetime.date):
+ dt = datetime.datetime(dt.year, dt.month, dt.day)
+ return df[df[time_col] <= dt], df[df[time_col] > dt]
+
+ horizon = 60
+ data_end = df.date.max()
+ train_end = data_end - datetime.timedelta(days=horizon)
+
+ train_df, val_df = split_by_date(df, train_end)
+ from flaml import AutoML
+
+ tgts = ["y"]
+ # tgt = "SERIES_SANCTIONS"
+
+ preds = {}
+ for tgt in tgts:
+ features = [] # [c for c in train_df.columns if "SERIES" not in c and c != time_col]
+
+ automl = AutoML(time_budget=5, metric="mae", task="ts_forecast", eval_method="cv")
+
+ automl.fit(
+ dataframe=train_df[[time_col] + features + [tgt]],
+ label=tgt,
+ period=horizon,
+ time_col=time_col,
+ verbose=4,
+ n_splits=5,
+ cv_step_size=5,
+ )
+
+ pred = automl.predict(val_df)
+
+ if isinstance(pred, pd.DataFrame):
+ pred = pred[tgt]
+ assert not np.isnan(pred.sum())
+
+ import matplotlib.pyplot as plt
+
+ preds[tgt] = pred
+ # plt.figure(figsize=(16, 8), dpi=80)
+ # plt.plot(df[time_col], df[tgt])
+ # plt.plot(val_df[time_col], pred)
+ # plt.legend(["actual", "predicted"])
+ # plt.show()
+
+ print("yahoo!")
+
+
if __name__ == "__main__":
- test_forecast_automl(60)
- test_multivariate_forecast_num(5)
- test_multivariate_forecast_cat(5)
- test_numpy()
- test_forecast_classification(5)
+ # test_forecast_automl(60)
+ # test_multivariate_forecast_num(5)
+ # test_multivariate_forecast_cat(5)
+ # test_numpy()
+ # test_forecast_classification(5)
test_forecast_panel(5)
+ # test_cv_step()
diff --git a/test/test_model.py b/test/test_model.py
index b05f5d58a..ab4d89397 100644
--- a/test/test_model.py
+++ b/test/test_model.py
@@ -10,10 +10,8 @@ from flaml.automl.model import (
CatBoostEstimator,
XGBoostEstimator,
RandomForestEstimator,
- Prophet,
- ARIMA,
- LGBM_TS,
)
+from flaml.automl.time_series import Prophet, ARIMA, LGBM_TS, TimeSeriesDataset
def test_lrl2():
@@ -101,7 +99,8 @@ def test_prep():
pass
prophet.predict(X)
- arima = ARIMA()
+ # What's the point of callin ARIMA without parameters, or calling predict before fit?
+ arima = ARIMA(p=1, q=1, d=0)
arima.predict(X)
arima._model = False
try:
@@ -109,23 +108,27 @@ def test_prep():
except ValueError:
# X_test needs to be either a pandas Dataframe with dates as the first column or an int number of periods for predict().
pass
-
- lgbm = LGBM_TS(optimize_for_horizon=True, lags=1)
+ lgbm = LGBM_TS(lags=1)
X = DataFrame(
{
"A": [
- datetime(1900, 2, 3),
+ datetime(1900, 3, 1),
+ datetime(1900, 3, 2),
+ datetime(1900, 3, 3),
datetime(1900, 3, 4),
datetime(1900, 3, 4),
datetime(1900, 3, 4),
- datetime(1900, 7, 2),
- datetime(1900, 8, 9),
+ datetime(1900, 3, 5),
+ datetime(1900, 3, 6),
],
}
)
- y = np.array([0, 1, 0, 1, 0, 0])
+ y = np.array([0, 1, 0, 1, 1, 1, 0, 0])
lgbm.predict(X[:2])
- lgbm.fit(X, y, period=2)
+ df = X.copy()
+ df["y"] = y
+ tsds = TimeSeriesDataset(df, time_col="A", target_names="y")
+ lgbm.fit(tsds, period=2)
lgbm.predict(X[:2])
print(lgbm.feature_names_in_)
print(lgbm.feature_importances_)