Skip to content

A programming framework for agentic AI

datetime feature engineering added. (#89)

Gian Pio Domizianic4c15f533fb1
diff --git a/flaml/data.py b/flaml/data.py
index c5f5f4de0..bef83d603 100644
--- a/flaml/data.py
+++ b/flaml/data.py
@@ -196,9 +196,6 @@ class DataTransformer:
drop = False
for column in X.columns:
# sklearn\utils\validation.py needs int/float values
- if X[column].dtype.name == 'datetime64[ns]':
- X[column] = X[column].map(datetime.toordinal)
- datetime_columns.append(column)
if X[column].dtype.name in ('object', 'category'):
if X[column].nunique() == 1 or X[column].nunique(
dropna=True) == n - X[column].isnull().sum():
@@ -219,8 +216,25 @@ class DataTransformer:
X.drop(columns=column, inplace=True)
drop = True
else:
- X[column] = X[column].fillna(np.nan)
- num_columns.append(column)
+ if X[column].dtype.name == 'datetime64[ns]':
+ tmp_dt = X[column].dt
+ new_columns_dict = {f'year_{column}': tmp_dt.year, f'month_{column}': tmp_dt.month,
+ f'day_{column}': tmp_dt.day, f'hour_{column}': tmp_dt.hour,
+ f'minute_{column}': tmp_dt.minute, f'second_{column}': tmp_dt.second,
+ f'dayofweek_{column}': tmp_dt.dayofweek,
+ f'dayofyear_{column}': tmp_dt.dayofyear,
+ f'quarter_{column}': tmp_dt.quarter}
+ for new_col_name in new_columns_dict.keys():
+ if new_col_name not in X.columns and \
+ new_columns_dict.get(new_col_name).nunique(dropna=False) >= 2:
+ X[new_col_name] = new_columns_dict.get(new_col_name)
+ num_columns.append(new_col_name)
+ X[column] = X[column].map(datetime.toordinal)
+ datetime_columns.append(column)
+ del tmp_dt
+ else:
+ X[column] = X[column].fillna(np.nan)
+ num_columns.append(column)
X = X[cat_columns + num_columns]
if cat_columns:
X[cat_columns] = X[cat_columns].astype('category')
@@ -254,12 +268,23 @@ class DataTransformer:
if isinstance(X, pd.DataFrame):
cat_columns, num_columns, datetime_columns = self._cat_columns, \
self._num_columns, self._datetime_columns
- X = X[cat_columns + num_columns].copy()
if datetime_columns:
- for dt_column in datetime_columns:
- X[dt_column] = X[dt_column].map(datetime.toordinal)
+ for column in datetime_columns:
+ tmp_dt = X[column].dt
+ new_columns_dict = {f'year_{column}': tmp_dt.year, f'month_{column}': tmp_dt.month,
+ f'day_{column}': tmp_dt.day, f'hour_{column}': tmp_dt.hour,
+ f'minute_{column}': tmp_dt.minute, f'second_{column}': tmp_dt.second,
+ f'dayofweek_{column}': tmp_dt.dayofweek,
+ f'dayofyear_{column}': tmp_dt.dayofyear,
+ f'quarter_{column}': tmp_dt.quarter}
+ for new_col_name in new_columns_dict.keys():
+ if new_col_name not in X.columns and \
+ new_columns_dict.get(new_col_name).nunique(dropna=False) >= 2:
+ X[new_col_name] = new_columns_dict.get(new_col_name)
+ X[column] = X[column].map(datetime.toordinal)
+ del tmp_dt
+ X = X[cat_columns + num_columns].copy()
for column in cat_columns:
- # print(column, X[column].dtype.name)
if X[column].dtype.name == 'object':
X[column] = X[column].fillna('__NAN__')
elif X[column].dtype.name == 'category':
diff --git a/flaml/ml.py b/flaml/ml.py
index a2decf098..b7c27a888 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -103,7 +103,7 @@ def sklearn_metric_loss_score(
raise ValueError(
metric_name + ' is not a built-in metric, '
'currently built-in metrics are: '
- 'r2, rmse, mae, mse, accuracy, roc_auc, log_loss, f1, ap. '
+ 'r2, rmse, mae, mse, accuracy, roc_auc, log_loss, f1, micro_f1, macro_f1, ap. '
'please pass a customized metric function to AutoML.fit(metric=func)')
return score
diff --git a/flaml/version.py b/flaml/version.py
index 6a9beea82..3d26edf77 100644
--- a/flaml/version.py
+++ b/flaml/version.py
@@ -1 +1 @@
-__version__ = "0.4.0"
+__version__ = "0.4.1"
diff --git a/test/test_automl.py b/test/test_automl.py
index 0122771d7..15211d919 100644
--- a/test/test_automl.py
+++ b/test/test_automl.py
@@ -150,7 +150,6 @@ class TestAutoML(unittest.TestCase):
self.test_classification(True)
def test_custom_metric(self):
-
X_train, y_train = load_iris(return_X_y=True)
automl_experiment = AutoML()
automl_settings = {
@@ -185,7 +184,6 @@ class TestAutoML(unittest.TestCase):
print(train_loss_history)
def test_classification(self, as_frame=False):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 4,
@@ -223,7 +221,6 @@ class TestAutoML(unittest.TestCase):
print(automl_experiment.predict_proba(X_train)[:5])
def test_datetime_columns(self):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 2,
@@ -234,19 +231,22 @@ class TestAutoML(unittest.TestCase):
"n_jobs": 1,
"model_history": True
}
-
- fake_df = pd.DataFrame({'A': [datetime(1900, 2, 3), datetime(1900, 3, 4)]})
- y = np.array([0, 1])
- automl_experiment.fit(
- X_train=fake_df, X_val=fake_df, y_train=y, y_val=y, **automl_settings)
-
- y_pred = automl_experiment.predict(fake_df)
- print(y_pred)
+ fake_df = pd.DataFrame({'A': [datetime(1900, 2, 3), datetime(1900, 3, 4),
+ datetime(1900, 3, 4), datetime(1900, 3, 4),
+ datetime(1900, 7, 2), datetime(1900, 8, 9)],
+ 'B': [datetime(1900, 1, 1), datetime(1900, 1, 1),
+ datetime(1900, 1, 1), datetime(1900, 1, 1),
+ datetime(1900, 1, 1), datetime(1900, 1, 1)],
+ 'year_A': [datetime(1900, 1, 2), datetime(1900, 8, 1),
+ datetime(1900, 1, 4), datetime(1900, 6, 1),
+ datetime(1900, 1, 5), datetime(1900, 4, 1)]})
+ y = np.array([0, 1, 0, 1, 0, 0])
+ automl_experiment.fit(X_train=fake_df, y_train=y, **automl_settings)
+ _ = automl_experiment.predict(fake_df)
def test_micro_macro_f1(self):
- automl_experiment = AutoML()
+ automl_experiment_micro = AutoML()
automl_experiment_macro = AutoML()
-
automl_settings = {
"time_budget": 2,
"task": 'classification',
@@ -255,15 +255,13 @@ class TestAutoML(unittest.TestCase):
"n_jobs": 1,
"model_history": True
}
-
X_train, y_train = load_iris(return_X_y=True)
- automl_experiment.fit(
+ automl_experiment_micro.fit(
X_train=X_train, y_train=y_train, metric='micro_f1', **automl_settings)
automl_experiment_macro.fit(
X_train=X_train, y_train=y_train, metric='macro_f1', **automl_settings)
def test_regression(self):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 2,
@@ -289,7 +287,6 @@ class TestAutoML(unittest.TestCase):
print(get_output_from_log(automl_settings["log_file_name"], 1))
def test_sparse_matrix_classification(self):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 2,
@@ -313,7 +310,6 @@ class TestAutoML(unittest.TestCase):
print(automl_experiment.best_estimator)
def test_sparse_matrix_regression(self):
-
X_train = scipy.sparse.random(300, 900, density=0.0001)
y_train = np.random.uniform(size=300)
X_val = scipy.sparse.random(100, 900, density=0.0001)
@@ -343,7 +339,6 @@ class TestAutoML(unittest.TestCase):
print(automl_experiment.best_config_train_time)
def test_sparse_matrix_xgboost(self):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 3,
@@ -366,7 +361,6 @@ class TestAutoML(unittest.TestCase):
print(automl_experiment.best_estimator)
def test_sparse_matrix_lr(self):
-
automl_experiment = AutoML()
automl_settings = {
"time_budget": 2,
@@ -389,7 +383,6 @@ class TestAutoML(unittest.TestCase):
print(automl_experiment.best_estimator)
def test_sparse_matrix_regression_cv(self):
-
X_train = scipy.sparse.random(8, 100)
y_train = np.random.uniform(size=8)
automl_experiment = AutoML()