A programming framework for agentic AI
datetime feature engineering (#285)
Chi Wangdb1fb9b47b52
diff --git a/flaml/data.py b/flaml/data.pyindex 41150213f..270cd048d 100644--- a/flaml/data.py+++ b/flaml/data.py@@ -269,40 +269,35 @@ class DataTransformer:else:X[column] = X[column].fillna("__NAN__")cat_columns.append(column)- else:- # print(X[column].dtype.name)- if X[column].nunique(dropna=True) < 2:- X.drop(columns=column, inplace=True)- drop = True- else:- if X[column].dtype.name == "datetime64[ns]":- tmp_dt = X[column].dt- new_columns_dict = {- f"year_{column}": tmp_dt.year,- f"month_{column}": tmp_dt.month,- f"day_{column}": tmp_dt.day,- f"hour_{column}": tmp_dt.hour,- f"minute_{column}": tmp_dt.minute,- f"second_{column}": tmp_dt.second,- f"dayofweek_{column}": tmp_dt.dayofweek,- f"dayofyear_{column}": tmp_dt.dayofyear,- f"quarter_{column}": tmp_dt.quarter,- }- for new_col_name in new_columns_dict.keys():- if (- new_col_name not in X.columns- and new_columns_dict.get(new_col_name).nunique(- dropna=False- )- >= 2- ):- X[new_col_name] = new_columns_dict.get(new_col_name)- num_columns.append(new_col_name)- X[column] = X[column].map(datetime.toordinal)- datetime_columns.append(column)- del tmp_dt- X[column] = X[column].fillna(np.nan)- num_columns.append(column)+ elif X[column].nunique(dropna=True) < 2:+ X.drop(columns=column, inplace=True)+ drop = True+ else: # datetime or numeric+ if X[column].dtype.name == "datetime64[ns]":+ tmp_dt = X[column].dt+ new_columns_dict = {+ f"year_{column}": tmp_dt.year,+ f"month_{column}": tmp_dt.month,+ f"day_{column}": tmp_dt.day,+ f"hour_{column}": tmp_dt.hour,+ f"minute_{column}": tmp_dt.minute,+ f"second_{column}": tmp_dt.second,+ f"dayofweek_{column}": tmp_dt.dayofweek,+ f"dayofyear_{column}": tmp_dt.dayofyear,+ f"quarter_{column}": tmp_dt.quarter,+ }+ for key, value in new_columns_dict.items():+ if (+ key not in X.columns+ and value.nunique(dropna=False) >= 2+ ):+ X[key] = value+ num_columns.append(key)+ X[column] = X[column].map(datetime.toordinal)+ datetime_columns.append(column)+ del tmp_dt+ X[column] = X[column].fillna(np.nan)+ num_columns.append(column)X = X[cat_columns + num_columns]if task == TS_FORECAST:X.insert(0, TS_TIMESTAMP_COL, ds_col)@@ -380,29 +375,24 @@ class DataTransformer:if self._task == TS_FORECAST:X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})ds_col = X.pop(TS_TIMESTAMP_COL)- if datetime_columns:- for column in datetime_columns:- tmp_dt = X[column].dt- new_columns_dict = {- f"year_{column}": tmp_dt.year,- f"month_{column}": tmp_dt.month,- f"day_{column}": tmp_dt.day,- f"hour_{column}": tmp_dt.hour,- f"minute_{column}": tmp_dt.minute,- f"second_{column}": tmp_dt.second,- f"dayofweek_{column}": tmp_dt.dayofweek,- f"dayofyear_{column}": tmp_dt.dayofyear,- f"quarter_{column}": tmp_dt.quarter,- }- for new_col_name in new_columns_dict.keys():- if (- new_col_name not in X.columns- and new_columns_dict.get(new_col_name).nunique(dropna=False)- >= 2- ):- X[new_col_name] = new_columns_dict.get(new_col_name)- X[column] = X[column].map(datetime.toordinal)- del tmp_dt+ for column in datetime_columns:+ tmp_dt = X[column].dt+ new_columns_dict = {+ f"year_{column}": tmp_dt.year,+ f"month_{column}": tmp_dt.month,+ f"day_{column}": tmp_dt.day,+ f"hour_{column}": tmp_dt.hour,+ f"minute_{column}": tmp_dt.minute,+ f"second_{column}": tmp_dt.second,+ f"dayofweek_{column}": tmp_dt.dayofweek,+ f"dayofyear_{column}": tmp_dt.dayofyear,+ f"quarter_{column}": tmp_dt.quarter,+ }+ for new_col_name, new_col_value in new_columns_dict.items():+ if new_col_name not in X.columns and new_col_name in num_columns:+ X[new_col_name] = new_col_value+ X[column] = X[column].map(datetime.toordinal)+ del tmp_dtX = X[cat_columns + num_columns].copy()if self._task == TS_FORECAST:X.insert(0, TS_TIMESTAMP_COL, ds_col)