A programming framework for agentic AI
Fixing the bug in custom metric (#356)
Xueqing Liudcfd2181086a
diff --git a/flaml/model.py b/flaml/model.pyindex 3e08cb214..c887d32c4 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -437,7 +437,7 @@ class TransformersEstimator(BaseEstimator):set_seed(self.params.get("seed", self._TrainingArguments.seed))self._init_hpo_args(kwargs)- self._metric_name = kwargs["metric"]+ self._metric = kwargs["metric"]if hasattr(self, "use_ray") is False:self.use_ray = kwargs["use_ray"]@@ -445,13 +445,14 @@ class TransformersEstimator(BaseEstimator):y_val = kwargs.get("y_val")if self._task not in NLG_TASKS:- X_train, _ = self._preprocess(X=X_train, task=self._task, **kwargs)+ self._X_train, _ = self._preprocess(X=X_train, task=self._task, **kwargs)+ self._y_train = y_trainelse:- X_train, y_train = self._preprocess(+ self._X_train, self._y_train = self._preprocess(X=X_train, y=y_train, task=self._task, **kwargs)- train_dataset = Dataset.from_pandas(self._join(X_train, y_train))+ train_dataset = Dataset.from_pandas(self._join(self._X_train, self._y_train))# TODO: set a breakpoint here, observe the resulting train_dataset,# compare it with the output of the tokenized results in your transformer example@@ -461,12 +462,13 @@ class TransformersEstimator(BaseEstimator):if X_val is not None:if self._task not in NLG_TASKS:- X_val, _ = self._preprocess(X=X_val, task=self._task, **kwargs)+ self._X_val, _ = self._preprocess(X=X_val, task=self._task, **kwargs)+ self._y_val = y_valelse:- X_val, y_val = self._preprocess(+ self._X_val, self._y_val = self._preprocess(X=X_val, y=y_val, task=self._task, **kwargs)- eval_dataset = Dataset.from_pandas(self._join(X_val, y_val))+ eval_dataset = Dataset.from_pandas(self._join(self._X_val, self._y_val))else:eval_dataset = None@@ -475,13 +477,13 @@ class TransformersEstimator(BaseEstimator):)self._tokenizer = tokenizer- num_labels = get_num_labels(self._task, y_train)+ num_labels = get_num_labels(self._task, self._y_train)training_args_config, per_model_config = separate_config(self.params, self._task)ckpt_freq = compute_checkpoint_freq(- train_data_size=len(X_train),+ train_data_size=len(self._X_train),custom_hpo_args=self.custom_hpo_args,num_train_epochs=training_args_config.get("num_train_epochs", self._TrainingArguments.num_train_epochs@@ -630,11 +632,22 @@ class TransformersEstimator(BaseEstimator):else np.argmax(predictions, axis=1))- return {- "val_loss": metric_loss_score(- metric_name=self._metric_name, y_predict=predictions, y_true=labels+ if isinstance(self._metric, str):+ return {+ "val_loss": metric_loss_score(+ metric_name=self._metric, y_predict=predictions, y_true=labels+ )+ }+ else:+ agg_metric, metric_dict = self._metric(+ X_test=self._X_val,+ y_test=self._y_val,+ estimator=self,+ labels=None,+ X_train=self._X_train,+ y_train=self._y_train,)- }+ return metric_dictdef predict_proba(self, X_test):assert (diff --git a/test/nlp/test_autohf_maxiter1.py b/test/nlp/test_autohf_custom_metric.pysimilarity index 66%rename from test/nlp/test_autohf_maxiter1.pyrename to test/nlp/test_autohf_custom_metric.pyindex c389b0df0..5c70d01d8 100644--- a/test/nlp/test_autohf_maxiter1.py+++ b/test/nlp/test_autohf_custom_metric.py@@ -2,8 +2,28 @@ import sysimport pytest+def toy_metric(+ X_test,+ y_test,+ estimator,+ labels,+ X_train,+ y_train,+ weight_test=None,+ weight_train=None,+ config=None,+ groups_test=None,+ groups_train=None,+):+ return 0, {+ "val_loss": 0,+ "train_loss": 0,+ "pred_time": 0,+ }++@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")-def test_max_iter_1():+def test_custom_metric():from flaml import AutoMLimport requestsfrom datasets import load_dataset@@ -27,24 +47,7 @@ def test_max_iter_1():automl = AutoML()- def toy_metric(- X_test,- y_test,- estimator,- labels,- X_train,- y_train,- weight_test=None,- weight_train=None,- config=None,- groups_test=None,- groups_train=None,- ):- return 0, {- "test_loss": 0,- "train_loss": 0,- "pred_time": 0,- }+ # testing when max_iter=1 and do retrain only without hpoautoml_settings = {"gpu_per_trial": 0,@@ -65,4 +68,12 @@ def test_max_iter_1():automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)++ # testing calling custom metric in TransformersEstimator._compute_metrics_by_dataset_name++ automl_settings["max_iter"] = 3+ automl.fit(+ X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings+ )+del automl