A programming framework for agentic AI
update
skzhang1fc633ef15e86
diff --git a/flaml/ml.py b/flaml/ml.pyindex 0c442d1c2..ec9a31063 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -427,6 +427,20 @@ def get_val_loss(train_time = time.time() - startreturn val_loss, metric_for_logging, train_time, pred_time+def default_cv_score_agg_func(metrics_across_folds):+ metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds)+ metrics_to_log = None+ for single_fold in metrics_across_folds:+ if single_fold[1] is None:+ break+ elif metrics_to_log is None:+ metrics_to_log = single_fold[1]+ else:+ metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()}+ if metrics_to_log:+ n = len(metrics_across_folds)+ metrics_to_log = {k: v / n for k, v in metrics_to_log.items()}+ return metric_to_minimize, metrics_to_logdef evaluate_model_CV(config,@@ -438,32 +452,18 @@ def evaluate_model_CV(task,eval_metric,best_val_loss,- cv_score_agg_func,+ cv_score_agg_func = None,log_training_metric=False,fit_kwargs={},):if cv_score_agg_func is None:- def cv_score_agg_func(metrics_across_folds):- metric_to_minimize = sum([tem[0] for tem in metrics_across_folds])/len(metrics_across_folds)- metrics_to_log = None- for single_fold in metrics_across_folds:- if single_fold[1] is None:- break- elif metrics_to_log is None:- metrics_to_log = single_fold[1]- else:- metrics_to_log = {k: metrics_to_log[k] + v for k, v in single_fold[1].items()}- if metrics_to_log:- n = len(metrics_across_folds)- metrics_to_log = {k: v / n for k, v in metrics_to_log.items()}- return metric_to_minimize, metrics_to_log+ cv_score_agg_func = default_cv_score_agg_funcstart_time = time.time()val_loss_folds = []log_metric_folds = []- total_metric = Nonemetric = Nonetrain_time = pred_time = 0- valid_fold_num = total_fold_num = 0+ total_fold_num = 0n = kf.get_n_splits()X_train_split, y_train_split = X_train_all, y_train_allif task in CLASSIFICATION:@@ -485,7 +485,6 @@ def evaluate_model_CV(else:kf = kf.split(X_train_split)rng = np.random.RandomState(2020)- val_loss_list = []budget_per_train = budget / nif "sample_weight" in fit_kwargs:weight = fit_kwargs["sample_weight"]@@ -530,32 +529,20 @@ def evaluate_model_CV()if weight is not None:fit_kwargs["sample_weight"] = weight- valid_fold_num += 1total_fold_num += 1val_loss_folds.append(val_loss_i)if log_training_metric or not isinstance(eval_metric, str):if isinstance(metric_i, dict):log_metric_folds.append(metric_i)- elif total_metric is not None:- total_metric += metric_i- else:- total_metric = metric_itrain_time += train_time_ipred_time += pred_time_i- if valid_fold_num == n:- val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0])- valid_fold_num = 0- val_loss_folds = []- elif time.time() - start_time >= budget:- val_loss_list.append(cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))[0])+ if time.time() - start_time >= budget:break- val_loss = np.max(val_loss_list)- n = total_fold_numif log_training_metric or not isinstance(eval_metric, str):- if len(log_metric_folds):- metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))[1]- else:- metric = total_metric / n+ val_loss, metric = cv_score_agg_func(list(zip([0]*len(log_metric_folds),log_metric_folds)))+ else:+ val_loss, metric = cv_score_agg_func(list(zip(val_loss_folds,[None]*len(val_loss_folds))))+ n = total_fold_numpred_time /= nreturn val_loss, metric, train_time, pred_time