Skip to content

A programming framework for agentic AI

micro/macro f1 metrics added. (#80)

Gian Pio Domiziani730fd14ef632
diff --git a/flaml/automl.py b/flaml/automl.py
index ab52821f0..042380438 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -809,7 +809,7 @@ class AutoML:
dataframe and label are ignored;
If not, dataframe and label must be provided.
metric: A string of the metric name or a function,
- e.g., 'accuracy', 'roc_auc', 'f1', 'log_loss', 'mae', 'mse', 'r2'
+ e.g., 'accuracy', 'roc_auc', 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2'
if passing a customized metric function, the function needs to
have the follwing signature:
@@ -902,7 +902,7 @@ class AutoML:
else:
metric = 'r2'
self._state.metric = metric
- if metric in ['r2', 'accuracy', 'roc_auc', 'f1', 'ap']:
+ if metric in ['r2', 'accuracy', 'roc_auc', 'f1', 'ap', 'micro_f1', 'macro_f1']:
error_metric = f"1-{metric}"
elif isinstance(metric, str):
error_metric = metric
diff --git a/flaml/ml.py b/flaml/ml.py
index 18c0971c2..8b939518d 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -54,9 +54,9 @@ def sklearn_metric_loss_score(
'''Loss using the specified metric
Args:
- metric_name: A string of the mtric name, one of
+ metric_name: A string of the metric name, one of
'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'log_loss',
- 'f1', 'ap'
+ 'f1', 'ap', 'micro_f1', 'macro_f1'
y_predict: A 1d or 2d numpy array of the predictions which can be
used to calculate the metric. E.g., 2d for log_loss and 1d
for others.
@@ -88,6 +88,10 @@ def sklearn_metric_loss_score(
elif 'log_loss' in metric_name:
score = log_loss(
y_true, y_predict, labels=labels, sample_weight=sample_weight)
+ elif 'micro_f1' in metric_name:
+ score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight, average='micro')
+ elif 'macro_f1' in metric_name:
+ score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight, average='macro')
elif 'f1' in metric_name:
score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight)
elif 'ap' in metric_name:
diff --git a/test/test_automl.py b/test/test_automl.py
index 94a4452c6..5d8a17e2e 100644
--- a/test/test_automl.py
+++ b/test/test_automl.py
@@ -241,6 +241,23 @@ class TestAutoML(unittest.TestCase):
y_pred = automl_experiment.predict(fake_df)
+ def test_micro_macro_f1(self):
+ automl_experiment = AutoML()
+ automl_experiment_macro = AutoML()
+
+ automl_settings = {
+ "time_budget": 2,
+ "task": 'classification',
+ "log_file_name": "test/micro_macro_f1.log",
+ "log_training_metric": True,
+ "n_jobs": 1,
+ "model_history": True
+ }
+
+ X_train, y_train = load_iris(return_X_y=True)
+ automl_experiment.fit(X_train=X_train, y_train=y_train, metric='micro_f1', **automl_settings)
+ automl_experiment_macro.fit(X_train=X_train, y_train=y_train, metric='macro_f1', **automl_settings)
+
def test_regression(self):
automl_experiment = AutoML()