A programming framework for agentic AI
moving intermediate_results logging from model.py to huggingface/trainer.py (#403)
Xueqing Liudda4ac90a1e7
diff --git a/flaml/model.py b/flaml/model.pyindex 61ea6b456..6dfdbc2b1 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -591,6 +591,8 @@ class TransformersEstimator(BaseEstimator):num_labels=self._num_labels,per_model_config=self._per_model_config,)+ if hasattr(self._trainer, "intermediate_results"):+ self._intermediate_results = self._trainer.intermediate_resultsself._trainer = Nonedef _delete_one_ckpt(self, ckpt_location):@@ -656,7 +658,7 @@ class TransformersEstimator(BaseEstimator):else np.argmax(predictions, axis=1))metric_dict = {- "val_loss": metric_loss_score(+ "automl_metric": metric_loss_score(metric_name=self._metric, y_predict=predictions, y_true=labels)}@@ -669,10 +671,7 @@ class TransformersEstimator(BaseEstimator):X_train=self._X_train,y_train=self._y_train,)- metric_dict["val_loss"] = loss- if not hasattr(self, "intermediate_results"):- self.intermediate_results = []- self.intermediate_results.append(metric_dict)+ metric_dict["automl_metric"] = lossreturn metric_dictdef _init_model_for_predict(self, X_test):diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/nlp/huggingface/trainer.pyindex 2bd81bf22..b657fc535 100644--- a/flaml/nlp/huggingface/trainer.py+++ b/flaml/nlp/huggingface/trainer.py@@ -74,6 +74,9 @@ class TrainerForAuto(Seq2SeqTrainer):ignore_keys,metric_key_prefix,)+ if not hasattr(self, "intermediate_results"):+ self.intermediate_results = []+ self.intermediate_results.append(metrics)# if metrics:# for key in list(metrics.keys()):# if key.startswith("eval_"):diff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.pyindex 861cdecd7..613857bd7 100644--- a/test/nlp/test_autohf_custom_metric.py+++ b/test/nlp/test_autohf_custom_metric.py@@ -36,7 +36,7 @@ def custom_metric(metrics = trainer.evaluate(eval_dataset)estimator._metric = estimator_metric_backup- return metrics.pop("eval_val_loss"), metrics+ return metrics.pop("eval_automl_metric"), metrics@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")