A programming framework for agentic AI
update config if n_estimators is modified (#225)
Chi Wanga99e939404ca
diff --git a/flaml/automl.py b/flaml/automl.pyindex b281cd7e6..d5441f23d 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -117,9 +117,17 @@ class SearchState:time2eval = result["time_total_s"]trained_estimator = result["trained_estimator"]del result["trained_estimator"] # free up RAM+ n_iter = trained_estimator and trained_estimator.params.get("n_estimators")+ if (+ n_iter is not None+ and "n_estimators" in config+ and n_iter >= self._search_space_domain["n_estimators"].lower+ ):+ config["n_estimators"] = n_iter+ n_iter = Noneelse:obj, time2eval, trained_estimator = np.inf, 0.0, None- metric_for_logging = config = None+ metric_for_logging = config = n_iter = Noneself.trial_time = time2evalself.total_time_used += time_usedself.total_iter += 1@@ -147,8 +155,10 @@ class SearchState:self.trained_estimator.cleanup()if trained_estimator:self.trained_estimator = trained_estimator+ self.best_n_iter = n_iterself.metric_for_logging = metric_for_loggingself.val_loss, self.config = obj, config+ self.n_iter = n_iterdef get_hist_config_sig(self, sample_size, config):config_values = tuple([config[k] for k in self._hp_names])@@ -251,7 +261,9 @@ class AutoMLState:# tune.report(**result)return result- def _train_with_config(self, estimator, config_w_resource, sample_size=None):+ def _train_with_config(+ self, estimator, config_w_resource, sample_size=None, n_iter=None+ ):if not sample_size:sample_size = config_w_resource.get("FLAML_sample_size", len(self.y_train_all)@@ -288,6 +300,7 @@ class AutoMLState:self.n_jobs,self.learner_classes.get(estimator),budget,+ n_iter,self.fit_kwargs,)if sampled_weight is not None:@@ -444,7 +457,9 @@ class AutoML:if y_pred.ndim > 1 and isinstance(y_pred, np.ndarray):y_pred = y_pred.flatten()if self._label_transformer:- return self._label_transformer.inverse_transform(pd.Series(y_pred))+ return self._label_transformer.inverse_transform(+ pd.Series(y_pred.astype(int))+ )else:return y_pred@@ -606,7 +621,7 @@ class AutoML:if (self._state.task in ("binary", "multi")and self._state.fit_kwargs.get("sample_weight") is None- and self._split_type != "time"+ and self._split_type not in ["time", "group"]):# logger.info(f"label {pd.unique(y_train_all)}")label_set, counts = np.unique(y_train_all, return_counts=True)@@ -695,12 +710,12 @@ class AutoML:test_size=split_ratio,shuffle=False,)- elif self._state.task == "rank":+ elif self._split_type == "group":gss = GroupShuffleSplit(n_splits=1, test_size=split_ratio, random_state=RANDOM_SEED)for train_idx, val_idx in gss.split(- X_train_all, y_train_all, self._state.groups+ X_train_all, y_train_all, self._state.groups_all):if self._df:X_train = X_train_all.iloc[train_idx]@@ -708,8 +723,8 @@ class AutoML:else:X_train, X_val = X_train_all[train_idx], X_train_all[val_idx]y_train, y_val = y_train_all[train_idx], y_train_all[val_idx]- self._state.groups = self._state.groups[train_idx]- self._state.groups_val = self._state.groups[val_idx]+ self._state.groups = self._state.groups_all[train_idx]+ self._state.groups_val = self._state.groups_all[val_idx]elif self._state.task in ("binary", "multi"):# for classification, make sure the labels are complete in both# training and validation data@@ -920,7 +935,7 @@ class AutoML:n_splits: An integer of the number of folds for cross-validation.split_type: str or None, default=None | the data split type.For classification tasks, valid choices are [- None, 'stratified', 'uniform', 'time']. None -> stratified.+ None, 'stratified', 'uniform', 'time', 'group']. None -> stratified.For regression tasks, valid choices are [None, 'uniform', 'time'].None -> uniform.For time series forecasting, must be None or 'time'.@@ -1007,7 +1022,7 @@ class AutoML:self._state.time_budget = Noneself._state.n_jobs = n_jobsself._trained_estimator = self._state._train_with_config(- best_estimator, best_config, sample_size+ best_estimator, best_config, sample_size, best.n_iter)[0]logger.info("retrain from log succeeded")return training_duration@@ -1018,10 +1033,12 @@ class AutoML:len(np.unique(self._y_train_all)))if self._state.task in ("binary", "multi"):- assert split_type in [None, "stratified", "uniform", "time"]- self._split_type = split_type or "stratified"+ assert split_type in [None, "stratified", "uniform", "time", "group"]+ self._split_type = (+ split_type or self._state.groups is None and "stratified" or "group"+ )elif self._state.task == "regression":- assert split_type in [None, "uniform", "time"]+ assert split_type in [None, "uniform", "time", "group"]self._split_type = split_type or "uniform"elif self._state.task == "forecast":assert split_type in [None, "time"]@@ -1420,15 +1437,16 @@ class AutoML:self.verbose = verboseif verbose == 0:logger.setLevel(logging.WARNING)- self._decide_split_type(split_type)- if eval_method == "auto" or self._state.X_val is not None:- eval_method = self._decide_eval_method(time_budget)- self._state.eval_method = eval_methodif (not mlflow or not mlflow.active_run()) and not logger.handlers:# Add the console handler._ch = logging.StreamHandler()_ch.setFormatter(logger_formatter)logger.addHandler(_ch)+ self._decide_split_type(split_type)+ logger.info(f"Data split method: {self._split_type}")+ if eval_method == "auto" or self._state.X_val is not None:+ eval_method = self._decide_eval_method(time_budget)+ self._state.eval_method = eval_methodlogger.info("Evaluation method: {}".format(eval_method))self._retrain_in_budget = retrain_full == "budget" and (@@ -1697,10 +1715,9 @@ class AutoML:self._state.time_from_start,search_state.val_loss,config,- self._state.best_loss,- search_state.best_config,estimator,search_state.sample_size,+ search_state.n_iter,)def _search_sequential(self):@@ -1909,10 +1926,9 @@ class AutoML:self._state.time_from_start,search_state.val_loss,search_state.config,- search_state.best_loss,- search_state.best_config,estimator,search_state.sample_size,+ search_state.n_iter,)if mlflow is not None and mlflow.active_run():with mlflow.start_run(nested=True):@@ -1985,10 +2001,12 @@ class AutoML:<= est_retrain_time + next_trial_time)):+ state = self._search_states[self._best_estimator]self._trained_estimator, retrain_time = self._state._train_with_config(self._best_estimator,- self._search_states[self._best_estimator].best_config,+ state.best_config,self.data_size_full,+ state.best_n_iter,)logger.info("retrain {} for {:.1f}s".format(self._best_estimator, retrain_time)@@ -2093,13 +2111,15 @@ class AutoML:> self._selected.est_retrain_time(self.data_size_full)and self._selected.best_config_sample_size == self._state.data_size):+ state = self._search_states[self._best_estimator](self._trained_estimator,retrain_time,) = self._state._train_with_config(self._best_estimator,- self._search_states[self._best_estimator].best_config,+ state.best_config,self.data_size_full,+ state.best_n_iter,)logger.info("retrain {} for {:.1f}s".format(diff --git a/flaml/ml.py b/flaml/ml.pyindex 37206ce4f..19ff9c26c 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -465,11 +465,14 @@ def train_estimator(n_jobs=1,estimator_class=None,budget=None,+ n_iter=None,fit_kwargs={},):start_time = time.time()estimator_class = estimator_class or get_estimator_class(task, estimator_name)estimator = estimator_class(**config_dic, task=task, n_jobs=n_jobs)+ if n_iter is not None:+ estimator.params["n_estimators"] = n_iterif X_train is not None:train_time = estimator.fit(X_train, y_train, budget, **fit_kwargs)else:diff --git a/flaml/model.py b/flaml/model.pyindex bf1092b27..cdce9812a 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -316,15 +316,18 @@ class LGBMEstimator(BaseEstimator):def fit(self, X_train, y_train, budget=None, **kwargs):start_time = time.time()n_iter = self.params["n_estimators"]+ trained = Falseif (- not self._time_per_iter or abs(self._train_size - X_train.shape[0]) > 4- ) and budget is not None:+ (not self._time_per_iter or abs(self._train_size - X_train.shape[0]) > 4)+ and budget is not None+ and n_iter > 1+ ):self.params["n_estimators"] = 1self._t1 = self._fit(X_train, y_train, **kwargs)if self._t1 >= budget:- self.params["n_estimators"] = n_iter+ # self.params["n_estimators"] = n_iterreturn self._t1- self.params["n_estimators"] = 4+ self.params["n_estimators"] = min(n_iter, 4)self._t2 = self._fit(X_train, y_train, **kwargs)self._time_per_iter = ((self._t2 - self._t1) / (self.params["n_estimators"] - 1)@@ -335,19 +338,24 @@ class LGBMEstimator(BaseEstimator):)self._train_size = X_train.shape[0]if self._t1 + self._t2 >= budget or n_iter == self.params["n_estimators"]:- self.params["n_estimators"] = n_iter+ # self.params["n_estimators"] = n_iterreturn time.time() - start_time- if budget is not None:- self.params["n_estimators"] = min(+ trained = True+ if budget is not None and n_iter > 1:+ max_iter = min(n_iter,int((budget - time.time() + start_time - self._t1) / self._time_per_iter+ 1),)+ if trained and max_iter <= self.params["n_estimators"]:+ return time.time() - start_time+ self.params["n_estimators"] = max_iterif self.params["n_estimators"] > 0:self._fit(X_train, y_train, **kwargs)- self.params["n_estimators"] = n_iter+ else:+ self.params["n_estimators"] = n_itertrain_time = time.time() - start_timereturn train_time@@ -787,10 +795,15 @@ class CatBoostEstimator(BaseEstimator):cat_features = []# from catboost import CatBoostError# try:+ trained = Falseif (- not CatBoostEstimator._time_per_iter- or abs(CatBoostEstimator._train_size - len(y_train)) > 4- ) and budget:+ (+ not CatBoostEstimator._time_per_iter+ or abs(CatBoostEstimator._train_size - len(y_train)) > 4+ )+ and budget+ and n_iter > 4+ ):# measure the time per iterationself.params["n_estimators"] = 1CatBoostEstimator._smallmodel = self.estimator_class(@@ -801,11 +814,11 @@ class CatBoostEstimator(BaseEstimator):)CatBoostEstimator._t1 = time.time() - start_timeif CatBoostEstimator._t1 >= budget:- self.params["n_estimators"] = n_iter+ # self.params["n_estimators"] = n_iterself._model = CatBoostEstimator._smallmodelshutil.rmtree(train_dir, ignore_errors=True)return CatBoostEstimator._t1- self.params["n_estimators"] = 4+ self.params["n_estimators"] = min(n_iter, 4)CatBoostEstimator._smallmodel = self.estimator_class(train_dir=train_dir, **self.params)@@ -822,13 +835,14 @@ class CatBoostEstimator(BaseEstimator):time.time() - start_time >= budgetor n_iter == self.params["n_estimators"]):- self.params["n_estimators"] = n_iter+ # self.params["n_estimators"] = n_iterself._model = CatBoostEstimator._smallmodelshutil.rmtree(train_dir, ignore_errors=True)return time.time() - start_time- if budget:+ trained = True+ if budget and n_iter > 4:train_times = 1- self.params["n_estimators"] = min(+ max_iter = min(n_iter,int((budget - time.time() + start_time - CatBoostEstimator._t1)@@ -838,6 +852,9 @@ class CatBoostEstimator(BaseEstimator):),)self._model = CatBoostEstimator._smallmodel+ if trained and max_iter <= self.params["n_estimators"]:+ return time.time() - start_time+ self.params["n_estimators"] = max_iterif self.params["n_estimators"] > 0:n = max(int(len(y_train) * 0.9), len(y_train) - 1000)X_tr, y_tr = X_train[:n], y_train[:n]@@ -863,9 +880,10 @@ class CatBoostEstimator(BaseEstimator):if weight is not None:kwargs["sample_weight"] = weightself._model = model+ else:+ self.params["n_estimators"] = n_iter# except CatBoostError:# self._model = None- self.params["n_estimators"] = n_itertrain_time = time.time() - start_timereturn train_timediff --git a/flaml/training_log.py b/flaml/training_log.pyindex b4061a9d8..923d60c80 100644--- a/flaml/training_log.py+++ b/flaml/training_log.py@@ -1,7 +1,7 @@-'''!- * Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved.+"""!+ * Copyright (c) Microsoft Corporation. All rights reserved.* Licensed under the MIT License.-'''+"""import jsonfrom typing import IO@@ -10,19 +10,19 @@ import warningsclass TrainingLogRecord(object):-- def __init__(self,- record_id: int,- iter_per_learner: int,- logged_metric: float,- trial_time: float,- wall_clock_time: float,- validation_loss,- config,- best_validation_loss,- best_config,- learner,- sample_size):+ def __init__(+ self,+ record_id: int,+ iter_per_learner: int,+ logged_metric: float,+ trial_time: float,+ wall_clock_time: float,+ validation_loss: float,+ config: dict,+ learner: str,+ sample_size: int,+ n_iter: int,+ ):self.record_id = record_idself.iter_per_learner = iter_per_learnerself.logged_metric = logged_metric@@ -30,10 +30,9 @@ class TrainingLogRecord(object):self.wall_clock_time = wall_clock_timeself.validation_loss = validation_lossself.config = config- self.best_validation_loss = best_validation_loss- self.best_config = best_configself.learner = learnerself.sample_size = sample_size+ self.n_iter = n_iter # n_estimators for catboostdef dump(self, fp: IO[str]):d = vars(self)@@ -49,75 +48,78 @@ class TrainingLogRecord(object):class TrainingLogCheckPoint(TrainingLogRecord):-def __init__(self, curr_best_record_id: int):self.curr_best_record_id = curr_best_record_idclass TrainingLogWriter(object):-def __init__(self, output_filename: str):self.output_filename = output_filenameself.file = Noneself.current_best_loss_record_id = None- self.current_best_loss = float('+inf')+ self.current_best_loss = float("+inf")self.current_sample_size = Noneself.current_record_id = 0def open(self):- self.file = open(self.output_filename, 'w')+ self.file = open(self.output_filename, "w")def append_open(self):- self.file = open(self.output_filename, 'a')-- def append(self,- it_counter: int,- train_loss: float,- trial_time: float,- wall_clock_time: float,- validation_loss,- config,- best_validation_loss,- best_config,- learner,- sample_size):+ self.file = open(self.output_filename, "a")++ def append(+ self,+ it_counter: int,+ train_loss: float,+ trial_time: float,+ wall_clock_time: float,+ validation_loss,+ config,+ learner,+ sample_size,+ n_iter,+ ):if self.file is None:raise IOError("Call open() to open the outpute file first.")if validation_loss is None:- raise ValueError('TEST LOSS NONE ERROR!!!')- record = TrainingLogRecord(self.current_record_id,- it_counter,- train_loss,- trial_time,- wall_clock_time,- validation_loss,- config,- best_validation_loss,- best_config,- learner,- sample_size)- if validation_loss < self.current_best_loss or \- validation_loss == self.current_best_loss and \- self.current_sample_size is not None and \- sample_size > self.current_sample_size:+ raise ValueError("TEST LOSS NONE ERROR!!!")+ record = TrainingLogRecord(+ self.current_record_id,+ it_counter,+ train_loss,+ trial_time,+ wall_clock_time,+ validation_loss,+ config,+ learner,+ sample_size,+ n_iter,+ )+ if (+ validation_loss < self.current_best_loss+ or validation_loss == self.current_best_loss+ and self.current_sample_size is not None+ and sample_size > self.current_sample_size+ ):self.current_best_loss = validation_lossself.current_sample_size = sample_sizeself.current_best_loss_record_id = self.current_record_idself.current_record_id += 1record.dump(self.file)- self.file.write('\n')+ self.file.write("\n")self.file.flush()def checkpoint(self):if self.file is None:raise IOError("Call open() to open the outpute file first.")if self.current_best_loss_record_id is None:- warnings.warn("checkpoint() called before any record is written, "- "skipped.")+ warnings.warn(+ "checkpoint() called before any record is written, " "skipped."+ )returnrecord = TrainingLogCheckPoint(self.current_best_loss_record_id)record.dump(self.file)- self.file.write('\n')+ self.file.write("\n")self.file.flush()def close(self):@@ -127,7 +129,6 @@ class TrainingLogWriter(object):class TrainingLogReader(object):-def __init__(self, filename: str):self.filename = filenameself.file = Nonediff --git a/flaml/version.py b/flaml/version.pyindex 7bbb2ef5c..4c513f3b5 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.6.5"+__version__ = "0.6.6"diff --git a/test/test_split.py b/test/test_split.pyindex 8e97b3dac..579022d19 100644--- a/test/test_split.py+++ b/test/test_split.py@@ -17,7 +17,7 @@ def _test(split_type):automl_settings = {"time_budget": 2,# "metric": 'accuracy',- "task": 'classification',+ "task": "classification","log_file_name": "test/{}.log".format(dataset),"model_history": True,"log_training_metric": True,@@ -28,13 +28,16 @@ def _test(split_type):X, y = fetch_openml(name=dataset, return_X_y=True)except (ArffException, ValueError):from sklearn.datasets import load_wine+X, y = load_wine(return_X_y=True)- if split_type != 'time':- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33,- random_state=42)+ if split_type != "time":+ X_train, X_test, y_train, y_test = train_test_split(+ X, y, test_size=0.33, random_state=42+ )else:- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33,- shuffle=False)+ X_train, X_test, y_train, y_test = train_test_split(+ X, y, test_size=0.33, shuffle=False+ )automl.fit(X_train=X_train, y_train=y_train, **automl_settings)pred = automl.predict(X_test)@@ -53,36 +56,45 @@ def test_time():def test_groups():from sklearn.externals._arff import ArffException+try:X, y = fetch_openml(name=dataset, return_X_y=True)except (ArffException, ValueError):from sklearn.datasets import load_wine+X, y = load_wine(return_X_y=True)import numpy as np+automl = AutoML()automl_settings = {"time_budget": 2,- "task": 'classification',+ "task": "classification","log_file_name": "test/{}.log".format(dataset),"model_history": True,"eval_method": "cv","groups": np.random.randint(low=0, high=10, size=len(y)),- "estimator_list": ['lgbm', 'rf', 'xgboost', 'kneighbor'], # list of ML learners+ "estimator_list": ["lgbm", "rf", "xgboost", "kneighbor"],"learner_selector": "roundrobin",}automl.fit(X, y, **automl_settings)+ automl_settings["eval_method"] = "holdout"+ automl.fit(X, y, **automl_settings)+def test_rank():from sklearn.externals._arff import ArffException+try:X, y = fetch_openml(name=dataset, return_X_y=True)except (ArffException, ValueError):from sklearn.datasets import load_wine+X, y = load_wine(return_X_y=True)y = y.cat.codesimport numpy as np+automl = AutoML()automl_settings = {"time_budget": 2,@@ -90,8 +102,9 @@ def test_rank():"log_file_name": "test/{}.log".format(dataset),"model_history": True,"eval_method": "cv",- "groups": np.array( # group labels- [0] * 200 + [1] * 200 + [2] * 200 + [3] * 200 + [4] * 100 + [5] * 100),+ "groups": np.array( # group labels+ [0] * 200 + [1] * 200 + [2] * 200 + [3] * 200 + [4] * 100 + [5] * 100+ ),"learner_selector": "roundrobin",}automl.fit(X, y, **automl_settings)@@ -100,10 +113,10 @@ def test_rank():automl_settings = {"time_budget": 2,"task": "rank",- "metric": "ndcg@5", # 5 can be replaced by any number+ "metric": "ndcg@5", # 5 can be replaced by any number"log_file_name": "test/{}.log".format(dataset),"model_history": True,- "groups": [200] * 4 + [100] * 2, # alternative way: group counts+ "groups": [200] * 4 + [100] * 2, # alternative way: group counts# "estimator_list": ['lgbm', 'xgboost'], # list of ML learners"learner_selector": "roundrobin",}