Skip to content

A programming framework for agentic AI

update

skzhang1fc633ef15e86
diff --git a/flaml/ml.py b/flaml/ml.py
index 0c442d1c2..ec9a31063 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -427,6 +427,20 @@ def get_val_loss(
train_time = time.time() - start
return val_loss, metric_for_logging, train_time, pred_time
+def default_cv_score_agg_func(metrics_across_folds):
+ metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds)
+ metrics_to_log = None
+ for single_fold in metrics_across_folds:
+ if single_fold[1] is None:
+ break
+ elif metrics_to_log is None:
+ metrics_to_log = single_fold[1]
+ else:
+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()}
+ if metrics_to_log:
+ n = len(metrics_across_folds)
+ metrics_to_log = {k: v / n for k, v in metrics_to_log.items()}
+ return metric_to_minimize, metrics_to_log
def evaluate_model_CV(
config,
@@ -438,32 +452,18 @@ def evaluate_model_CV(
task,
eval_metric,
best_val_loss,
- cv_score_agg_func,
+ cv_score_agg_func = None,
log_training_metric=False,
fit_kwargs={},
):
if cv_score_agg_func is None:
- def cv_score_agg_func(metrics_across_folds):
- metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds)
- metrics_to_log = None
- for single_fold in metrics_across_folds:
- if single_fold[1] is None:
- break
- elif metrics_to_log is None:
- metrics_to_log = single_fold[1]
- else:
- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()}
- if metrics_to_log:
- n = len(metrics_across_folds)
- metrics_to_log = {k: v / n for k, v in metrics_to_log.items()}
- return metric_to_minimize, metrics_to_log
+ cv_score_agg_func = default_cv_score_agg_func
start_time = time.time()
val_loss_folds = []
log_metric_folds = []
- total_metric = None
metric = None
train_time = pred_time = 0
- valid_fold_num = total_fold_num = 0
+ total_fold_num = 0
n = kf.get_n_splits()
X_train_split, y_train_split = X_train_all, y_train_all
if task in CLASSIFICATION:
@@ -485,7 +485,6 @@ def evaluate_model_CV(
else:
kf = kf.split(X_train_split)
rng = np.random.RandomState(2020)
- val_loss_list = []
budget_per_train = budget / n
if "sample_weight" in fit_kwargs:
weight = fit_kwargs["sample_weight"]
@@ -530,32 +529,20 @@ def evaluate_model_CV(
)
if weight is not None:
fit_kwargs["sample_weight"] = weight
- valid_fold_num += 1
total_fold_num += 1
val_loss_folds.append(val_loss_i)
if log_training_metric or not isinstance(eval_metric, str):
if isinstance(metric_i, dict):
log_metric_folds.append(metric_i)
- elif total_metric is not None:
- total_metric += metric_i
- else:
- total_metric = metric_i
train_time += train_time_i
pred_time += pred_time_i
- if valid_fold_num == n:
- val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0])
- valid_fold_num = 0
- val_loss_folds = []
- elif time.time() - start_time >= budget:
- val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0])
+ if time.time() - start_time >= budget:
break
- val_loss = np.max(val_loss_list)
- n = total_fold_num
if log_training_metric or not isinstance(eval_metric, str):
- if len(log_metric_folds):
- metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))[1]
- else:
- metric = total_metric / n
+ val_loss, metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))
+ else:
+ val_loss, metric = cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))
+ n = total_fold_num
pred_time /= n
return val_loss, metric, train_time, pred_time