A programming framework for agentic AI
datetime feature engineering added. (#89)
Gian Pio Domizianic4c15f533fb1
diff --git a/flaml/data.py b/flaml/data.pyindex c5f5f4de0..bef83d603 100644--- a/flaml/data.py+++ b/flaml/data.py@@ -196,9 +196,6 @@ class DataTransformer:drop = Falsefor column in X.columns:# sklearn\utils\validation.py needs int/float values- if X[column].dtype.name == 'datetime64[ns]':- X[column] = X[column].map(datetime.toordinal)- datetime_columns.append(column)if X[column].dtype.name in ('object', 'category'):if X[column].nunique() == 1 or X[column].nunique(dropna=True) == n - X[column].isnull().sum():@@ -219,8 +216,25 @@ class DataTransformer:X.drop(columns=column, inplace=True)drop = Trueelse:- X[column] = X[column].fillna(np.nan)- num_columns.append(column)+ if X[column].dtype.name == 'datetime64[ns]':+ tmp_dt = X[column].dt+ new_columns_dict = {f'year_{column}': tmp_dt.year, f'month_{column}': tmp_dt.month,+ f'day_{column}': tmp_dt.day, f'hour_{column}': tmp_dt.hour,+ f'minute_{column}': tmp_dt.minute, f'second_{column}': tmp_dt.second,+ f'dayofweek_{column}': tmp_dt.dayofweek,+ f'dayofyear_{column}': tmp_dt.dayofyear,+ f'quarter_{column}': tmp_dt.quarter}+ for new_col_name in new_columns_dict.keys():+ if new_col_name not in X.columns and \+ new_columns_dict.get(new_col_name).nunique(dropna=False) >= 2:+ X[new_col_name] = new_columns_dict.get(new_col_name)+ num_columns.append(new_col_name)+ X[column] = X[column].map(datetime.toordinal)+ datetime_columns.append(column)+ del tmp_dt+ else:+ X[column] = X[column].fillna(np.nan)+ num_columns.append(column)X = X[cat_columns + num_columns]if cat_columns:X[cat_columns] = X[cat_columns].astype('category')@@ -254,12 +268,23 @@ class DataTransformer:if isinstance(X, pd.DataFrame):cat_columns, num_columns, datetime_columns = self._cat_columns, \self._num_columns, self._datetime_columns- X = X[cat_columns + num_columns].copy()if datetime_columns:- for dt_column in datetime_columns:- X[dt_column] = X[dt_column].map(datetime.toordinal)+ for column in datetime_columns:+ tmp_dt = X[column].dt+ new_columns_dict = {f'year_{column}': tmp_dt.year, f'month_{column}': tmp_dt.month,+ f'day_{column}': tmp_dt.day, f'hour_{column}': tmp_dt.hour,+ f'minute_{column}': tmp_dt.minute, f'second_{column}': tmp_dt.second,+ f'dayofweek_{column}': tmp_dt.dayofweek,+ f'dayofyear_{column}': tmp_dt.dayofyear,+ f'quarter_{column}': tmp_dt.quarter}+ for new_col_name in new_columns_dict.keys():+ if new_col_name not in X.columns and \+ new_columns_dict.get(new_col_name).nunique(dropna=False) >= 2:+ X[new_col_name] = new_columns_dict.get(new_col_name)+ X[column] = X[column].map(datetime.toordinal)+ del tmp_dt+ X = X[cat_columns + num_columns].copy()for column in cat_columns:- # print(column, X[column].dtype.name)if X[column].dtype.name == 'object':X[column] = X[column].fillna('__NAN__')elif X[column].dtype.name == 'category':diff --git a/flaml/ml.py b/flaml/ml.pyindex a2decf098..b7c27a888 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -103,7 +103,7 @@ def sklearn_metric_loss_score(raise ValueError(metric_name + ' is not a built-in metric, ''currently built-in metrics are: '- 'r2, rmse, mae, mse, accuracy, roc_auc, log_loss, f1, ap. '+ 'r2, rmse, mae, mse, accuracy, roc_auc, log_loss, f1, micro_f1, macro_f1, ap. ''please pass a customized metric function to AutoML.fit(metric=func)')return scorediff --git a/flaml/version.py b/flaml/version.pyindex 6a9beea82..3d26edf77 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.4.0"+__version__ = "0.4.1"diff --git a/test/test_automl.py b/test/test_automl.pyindex 0122771d7..15211d919 100644--- a/test/test_automl.py+++ b/test/test_automl.py@@ -150,7 +150,6 @@ class TestAutoML(unittest.TestCase):self.test_classification(True)def test_custom_metric(self):-X_train, y_train = load_iris(return_X_y=True)automl_experiment = AutoML()automl_settings = {@@ -185,7 +184,6 @@ class TestAutoML(unittest.TestCase):print(train_loss_history)def test_classification(self, as_frame=False):-automl_experiment = AutoML()automl_settings = {"time_budget": 4,@@ -223,7 +221,6 @@ class TestAutoML(unittest.TestCase):print(automl_experiment.predict_proba(X_train)[:5])def test_datetime_columns(self):-automl_experiment = AutoML()automl_settings = {"time_budget": 2,@@ -234,19 +231,22 @@ class TestAutoML(unittest.TestCase):"n_jobs": 1,"model_history": True}-- fake_df = pd.DataFrame({'A': [datetime(1900, 2, 3), datetime(1900, 3, 4)]})- y = np.array([0, 1])- automl_experiment.fit(- X_train=fake_df, X_val=fake_df, y_train=y, y_val=y, **automl_settings)-- y_pred = automl_experiment.predict(fake_df)- print(y_pred)+ fake_df = pd.DataFrame({'A': [datetime(1900, 2, 3), datetime(1900, 3, 4),+ datetime(1900, 3, 4), datetime(1900, 3, 4),+ datetime(1900, 7, 2), datetime(1900, 8, 9)],+ 'B': [datetime(1900, 1, 1), datetime(1900, 1, 1),+ datetime(1900, 1, 1), datetime(1900, 1, 1),+ datetime(1900, 1, 1), datetime(1900, 1, 1)],+ 'year_A': [datetime(1900, 1, 2), datetime(1900, 8, 1),+ datetime(1900, 1, 4), datetime(1900, 6, 1),+ datetime(1900, 1, 5), datetime(1900, 4, 1)]})+ y = np.array([0, 1, 0, 1, 0, 0])+ automl_experiment.fit(X_train=fake_df, y_train=y, **automl_settings)+ _ = automl_experiment.predict(fake_df)def test_micro_macro_f1(self):- automl_experiment = AutoML()+ automl_experiment_micro = AutoML()automl_experiment_macro = AutoML()-automl_settings = {"time_budget": 2,"task": 'classification',@@ -255,15 +255,13 @@ class TestAutoML(unittest.TestCase):"n_jobs": 1,"model_history": True}-X_train, y_train = load_iris(return_X_y=True)- automl_experiment.fit(+ automl_experiment_micro.fit(X_train=X_train, y_train=y_train, metric='micro_f1', **automl_settings)automl_experiment_macro.fit(X_train=X_train, y_train=y_train, metric='macro_f1', **automl_settings)def test_regression(self):-automl_experiment = AutoML()automl_settings = {"time_budget": 2,@@ -289,7 +287,6 @@ class TestAutoML(unittest.TestCase):print(get_output_from_log(automl_settings["log_file_name"], 1))def test_sparse_matrix_classification(self):-automl_experiment = AutoML()automl_settings = {"time_budget": 2,@@ -313,7 +310,6 @@ class TestAutoML(unittest.TestCase):print(automl_experiment.best_estimator)def test_sparse_matrix_regression(self):-X_train = scipy.sparse.random(300, 900, density=0.0001)y_train = np.random.uniform(size=300)X_val = scipy.sparse.random(100, 900, density=0.0001)@@ -343,7 +339,6 @@ class TestAutoML(unittest.TestCase):print(automl_experiment.best_config_train_time)def test_sparse_matrix_xgboost(self):-automl_experiment = AutoML()automl_settings = {"time_budget": 3,@@ -366,7 +361,6 @@ class TestAutoML(unittest.TestCase):print(automl_experiment.best_estimator)def test_sparse_matrix_lr(self):-automl_experiment = AutoML()automl_settings = {"time_budget": 2,@@ -389,7 +383,6 @@ class TestAutoML(unittest.TestCase):print(automl_experiment.best_estimator)def test_sparse_matrix_regression_cv(self):-X_train = scipy.sparse.random(8, 100)y_train = np.random.uniform(size=8)automl_experiment = AutoML()