Skip to content

A programming framework for agentic AI

multiple logged metrics in cv (#114)

Chi Wange039861ab0f4
diff --git a/flaml/ml.py b/flaml/ml.py
index afdb35952..c9606e62d 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -233,7 +233,10 @@ def evaluate_model_CV(
valid_fold_num += 1
total_val_loss += val_loss_i
if train_loss is not False:
- if total_train_loss != 0:
+ if isinstance(total_train_loss, list):
+ total_train_loss = [
+ total_train_loss[i] + v for i, v in enumerate(train_loss_i)]
+ elif total_train_loss != 0:
total_train_loss += train_loss_i
else:
total_train_loss = train_loss_i
@@ -246,7 +249,10 @@ def evaluate_model_CV(
break
val_loss = np.max(val_loss_list)
if train_loss is not False:
- train_loss = total_train_loss / n
+ if isinstance(total_train_loss, list):
+ train_loss = [v / n for v in total_train_loss]
+ else:
+ train_loss = total_train_loss / n
budget -= time.time() - start_time
if val_loss < best_val_loss and budget > budget_per_train:
estimator.cleanup()
diff --git a/test/test_automl.py b/test/test_automl.py
index 6916ba9ec..e8b7fece6 100644
--- a/test/test_automl.py
+++ b/test/test_automl.py
@@ -153,8 +153,8 @@ class TestAutoML(unittest.TestCase):
X_train, y_train = load_iris(return_X_y=True)
automl_experiment = AutoML()
automl_settings = {
- "time_budget": 10,
- 'eval_method': 'holdout',
+ "time_budget": 5,
+ 'eval_method': 'cv',
"metric": custom_metric,
"task": 'classification',
"log_file_name": "test/iris_custom.log",