A programming framework for agentic AI
n_estimators for catboost
Chi Wang7d6e8601026f
diff --git a/flaml/automl.py b/flaml/automl.pyindex 08cd66519..bef754535 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -89,6 +89,7 @@ class SearchState:and starting_point.get(name) is not None):self.init_config[name] = starting_point[name]+if isinstance(starting_point, list):self.init_config = starting_pointself._hp_names = list(self._search_space_domain.keys())@@ -104,7 +105,6 @@ class SearchState:self.trained_estimator = Noneself.sample_size = Noneself.trial_time = 0- self.best_n_iter = Nonedef update(self, result, time_used, save_model_history=False):if result:@@ -122,13 +122,12 @@ class SearchState:if (n_iter is not Noneand "n_estimators" in config- and n_iter >= self._search_space_domain["n_estimators"].lower+ # and n_iter >= self._search_space_domain["n_estimators"].lower):config["n_estimators"] = n_iter- n_iter = Noneelse:obj, time2eval, trained_estimator = np.inf, 0.0, None- metric_for_logging = config = n_iter = None+ metric_for_logging = config = Noneself.trial_time = time2evalself.total_time_used += time_usedself.total_iter += 1@@ -156,10 +155,8 @@ class SearchState:self.trained_estimator.cleanup()if trained_estimator:self.trained_estimator = trained_estimator- self.best_n_iter = n_iterself.metric_for_logging = metric_for_loggingself.val_loss, self.config = obj, config- self.n_iter = n_iterdef get_hist_config_sig(self, sample_size, config):config_values = tuple([config[k] for k in self._hp_names])@@ -262,9 +259,7 @@ class AutoMLState:# tune.report(**result)return result- def _train_with_config(- self, estimator, config_w_resource, sample_size=None, n_iter=None- ):+ def _train_with_config(self, estimator, config_w_resource, sample_size=None):if not sample_size:sample_size = config_w_resource.get("FLAML_sample_size", len(self.y_train_all)@@ -301,7 +296,6 @@ class AutoMLState:self.n_jobs,self.learner_classes.get(estimator),budget,- n_iter,self.fit_kwargs,)if sampled_weight is not None:@@ -1030,7 +1024,7 @@ class AutoML:self._state.time_budget = Noneself._state.n_jobs = n_jobsself._trained_estimator = self._state._train_with_config(- best_estimator, best_config, sample_size, best.n_iter+ best_estimator, best_config, sample_size)[0]logger.info("retrain from log succeeded")return training_duration@@ -1731,7 +1725,6 @@ class AutoML:config,estimator,search_state.sample_size,- search_state.n_iter,)def _search_sequential(self):@@ -1953,7 +1946,6 @@ class AutoML:search_state.config,estimator,search_state.sample_size,- search_state.n_iter,)if mlflow is not None and mlflow.active_run():with mlflow.start_run(nested=True):@@ -2031,7 +2023,6 @@ class AutoML:self._best_estimator,state.best_config,self.data_size_full,- state.best_n_iter,)logger.info("retrain {} for {:.1f}s".format(self._best_estimator, retrain_time)@@ -2144,7 +2135,6 @@ class AutoML:self._best_estimator,state.best_config,self.data_size_full,- state.best_n_iter,)logger.info("retrain {} for {:.1f}s".format(diff --git a/flaml/ml.py b/flaml/ml.pyindex b2d149565..3be9d8daf 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -465,14 +465,11 @@ def train_estimator(n_jobs=1,estimator_class=None,budget=None,- n_iter=None,fit_kwargs={},):start_time = time.time()estimator_class = estimator_class or get_estimator_class(task, estimator_name)estimator = estimator_class(**config_dic, task=task, n_jobs=n_jobs)- if n_iter is not None:- estimator.params["n_estimators"] = n_iterif X_train is not None:train_time = estimator.fit(X_train, y_train, budget, **fit_kwargs)else:diff --git a/flaml/model.py b/flaml/model.pyindex 2deaa4ccf..4de2fc2e0 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -645,11 +645,15 @@ class CatBoostEstimator(BaseEstimator):"domain": tune.loguniform(lower=0.005, upper=0.2),"init_value": 0.1,},+ "n_estimators": {+ "domain": 8192,+ "init_value": 8192,+ },}@classmethoddef size(cls, config):- n_estimators = 8192+ n_estimators = config.get("n_estimators", 8192)max_leaves = 64return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8diff --git a/flaml/training_log.py b/flaml/training_log.pyindex bcc9b43c9..a71488f03 100644--- a/flaml/training_log.py+++ b/flaml/training_log.py@@ -23,7 +23,6 @@ class TrainingLogRecord(object):config: dict,learner: str,sample_size: int,- n_iter: int,):self.record_id = record_idself.iter_per_learner = iter_per_learner@@ -34,7 +33,6 @@ class TrainingLogRecord(object):self.config = configself.learner = learnerself.sample_size = sample_size- self.n_iter = n_iter # n_estimators for catboostdef dump(self, fp: IO[str]):d = vars(self)@@ -79,7 +77,6 @@ class TrainingLogWriter(object):config,learner,sample_size,- n_iter,):if self.file is None:raise IOError("Call open() to open the outpute file first.")@@ -95,7 +92,6 @@ class TrainingLogWriter(object):config,learner,sample_size,- n_iter,)if (validation_loss < self.current_best_lossdiff --git a/test/test_training_log.py b/test/test_training_log.pyindex 69c435ef1..ac8fba170 100644--- a/test/test_training_log.py+++ b/test/test_training_log.py@@ -9,7 +9,7 @@ from flaml.training_log import training_log_readerclass TestTrainingLog(unittest.TestCase):- def test_training_log(self, path="test_training_log.log"):+ def test_training_log(self, path="test_training_log.log", estimator_list="auto"):with TemporaryDirectory() as d:filename = os.path.join(d, path)@@ -27,8 +27,9 @@ class TestTrainingLog(unittest.TestCase):"model_history": True,"train_time_limit": 0.1,"verbose": 3,- "ensemble": True,+ # "ensemble": True,"keep_search_state": True,+ "estimator_list": estimator_list,}X_train, y_train = fetch_california_housing(return_X_y=True)automl.fit(X_train=X_train, y_train=y_train, **automl_settings)@@ -37,31 +38,34 @@ class TestTrainingLog(unittest.TestCase):if automl.best_estimator:estimator, config = automl.best_estimator, automl.best_configmodel0 = automl.best_model_for_estimator(estimator)- print(model0.params["n_estimators"], model0.estimator)+ print(model0.params["n_estimators"], config)+ # train on full data with no time limitautoml._state.time_budget = Nonemodel, _ = automl._state._train_with_config(estimator, config)- print(model.estimator)- # model0 and model are equivalent unless model0's n_estimator is out of search space range- assert (- str(model0.estimator) == str(model.estimator)- or model0.params["n_estimators"] < 4- )# assuming estimator & config are saved and loaded as followsautoml = AutoML()automl.fit(X_train=X_train,y_train=y_train,- max_iter=0,+ max_iter=1,task="regression",estimator_list=[estimator],n_jobs=1,starting_points={estimator: config},)+ print(automl.best_config)# then the fitted model should be equivalent to model- # print(str(model.estimator), str(automl.model.estimator))- assert str(model.estimator) == str(automl.model.estimator)+ assert (+ str(model.estimator) == str(automl.model.estimator)+ or estimator == "xgboost"+ and str(model.estimator.get_dump())+ == str(automl.model.estimator.get_dump())+ or estimator == "catboost"+ and str(model.estimator.get_all_params())+ == str(automl.model.estimator.get_all_params())+ )with training_log_reader(filename) as reader:count = 0@@ -83,3 +87,10 @@ class TestTrainingLog(unittest.TestCase):print("IsADirectoryError happens as expected in linux.")except PermissionError:print("PermissionError happens as expected in windows.")++ def test_each_estimator(self):+ self.test_training_log(estimator_list=["xgboost"])+ self.test_training_log(estimator_list=["catboost"])+ self.test_training_log(estimator_list=["extra_tree"])+ self.test_training_log(estimator_list=["rf"])+ self.test_training_log(estimator_list=["lgbm"])