Skip to content

A programming framework for agentic AI

Fixing the bug in custom metric (#356)

Xueqing Liudcfd2181086a
diff --git a/flaml/model.py b/flaml/model.py
index 3e08cb214..c887d32c4 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -437,7 +437,7 @@ class TransformersEstimator(BaseEstimator):
set_seed(self.params.get("seed", self._TrainingArguments.seed))
self._init_hpo_args(kwargs)
- self._metric_name = kwargs["metric"]
+ self._metric = kwargs["metric"]
if hasattr(self, "use_ray") is False:
self.use_ray = kwargs["use_ray"]
@@ -445,13 +445,14 @@ class TransformersEstimator(BaseEstimator):
y_val = kwargs.get("y_val")
if self._task not in NLG_TASKS:
- X_train, _ = self._preprocess(X=X_train, task=self._task, **kwargs)
+ self._X_train, _ = self._preprocess(X=X_train, task=self._task, **kwargs)
+ self._y_train = y_train
else:
- X_train, y_train = self._preprocess(
+ self._X_train, self._y_train = self._preprocess(
X=X_train, y=y_train, task=self._task, **kwargs
)
- train_dataset = Dataset.from_pandas(self._join(X_train, y_train))
+ train_dataset = Dataset.from_pandas(self._join(self._X_train, self._y_train))
# TODO: set a breakpoint here, observe the resulting train_dataset,
# compare it with the output of the tokenized results in your transformer example
@@ -461,12 +462,13 @@ class TransformersEstimator(BaseEstimator):
if X_val is not None:
if self._task not in NLG_TASKS:
- X_val, _ = self._preprocess(X=X_val, task=self._task, **kwargs)
+ self._X_val, _ = self._preprocess(X=X_val, task=self._task, **kwargs)
+ self._y_val = y_val
else:
- X_val, y_val = self._preprocess(
+ self._X_val, self._y_val = self._preprocess(
X=X_val, y=y_val, task=self._task, **kwargs
)
- eval_dataset = Dataset.from_pandas(self._join(X_val, y_val))
+ eval_dataset = Dataset.from_pandas(self._join(self._X_val, self._y_val))
else:
eval_dataset = None
@@ -475,13 +477,13 @@ class TransformersEstimator(BaseEstimator):
)
self._tokenizer = tokenizer
- num_labels = get_num_labels(self._task, y_train)
+ num_labels = get_num_labels(self._task, self._y_train)
training_args_config, per_model_config = separate_config(
self.params, self._task
)
ckpt_freq = compute_checkpoint_freq(
- train_data_size=len(X_train),
+ train_data_size=len(self._X_train),
custom_hpo_args=self.custom_hpo_args,
num_train_epochs=training_args_config.get(
"num_train_epochs", self._TrainingArguments.num_train_epochs
@@ -630,11 +632,22 @@ class TransformersEstimator(BaseEstimator):
else np.argmax(predictions, axis=1)
)
- return {
- "val_loss": metric_loss_score(
- metric_name=self._metric_name, y_predict=predictions, y_true=labels
+ if isinstance(self._metric, str):
+ return {
+ "val_loss": metric_loss_score(
+ metric_name=self._metric, y_predict=predictions, y_true=labels
+ )
+ }
+ else:
+ agg_metric, metric_dict = self._metric(
+ X_test=self._X_val,
+ y_test=self._y_val,
+ estimator=self,
+ labels=None,
+ X_train=self._X_train,
+ y_train=self._y_train,
)
- }
+ return metric_dict
def predict_proba(self, X_test):
assert (
diff --git a/test/nlp/test_autohf_maxiter1.py b/test/nlp/test_autohf_custom_metric.py
similarity index 66%
rename from test/nlp/test_autohf_maxiter1.py
rename to test/nlp/test_autohf_custom_metric.py
index c389b0df0..5c70d01d8 100644
--- a/test/nlp/test_autohf_maxiter1.py
+++ b/test/nlp/test_autohf_custom_metric.py
@@ -2,8 +2,28 @@ import sys
import pytest
+def toy_metric(
+ X_test,
+ y_test,
+ estimator,
+ labels,
+ X_train,
+ y_train,
+ weight_test=None,
+ weight_train=None,
+ config=None,
+ groups_test=None,
+ groups_train=None,
+):
+ return 0, {
+ "val_loss": 0,
+ "train_loss": 0,
+ "pred_time": 0,
+ }
+
+
@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")
-def test_max_iter_1():
+def test_custom_metric():
from flaml import AutoML
import requests
from datasets import load_dataset
@@ -27,24 +47,7 @@ def test_max_iter_1():
automl = AutoML()
- def toy_metric(
- X_test,
- y_test,
- estimator,
- labels,
- X_train,
- y_train,
- weight_test=None,
- weight_train=None,
- config=None,
- groups_test=None,
- groups_train=None,
- ):
- return 0, {
- "test_loss": 0,
- "train_loss": 0,
- "pred_time": 0,
- }
+ # testing when max_iter=1 and do retrain only without hpo
automl_settings = {
"gpu_per_trial": 0,
@@ -65,4 +68,12 @@ def test_max_iter_1():
automl.fit(
X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings
)
+
+ # testing calling custom metric in TransformersEstimator._compute_metrics_by_dataset_name
+
+ automl_settings["max_iter"] = 3
+ automl.fit(
+ X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings
+ )
+
del automl