A programming framework for agentic AI
v0.5.12 (#150)
Qingyun Wu10082b9262a8
diff --git a/README.md b/README.mdindex b1cfee06c..71884d9df 100644--- a/README.md+++ b/README.md@@ -133,7 +133,7 @@ Please find demo and tutorials of FLAML [here](https://www.youtube.com/channel/UFor more technical details, please check our papers.-* [FLAML: A Fast and Lightweight AutoML Library](https://www.microsoft.com/en-us/research/publication/flaml-a-fast-and-lightweight-automl-library/). Chi Wang, Qingyun Wu, Markus Weimer, Erkang Zhu. MLSys, 2021.+* [FLAML: A Fast and Lightweight AutoML Library](https://www.microsoft.com/en-us/research/publication/flaml-a-fast-and-lightweight-automl-library/). Chi Wang, Qingyun Wu, Markus Weimer, Erkang Zhu. MLSys 2021.```bibtex@inproceedings{wang2021flaml,diff --git a/flaml/automl.py b/flaml/automl.pyindex 1e4fb3aa4..9e3f78169 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -13,8 +13,6 @@ from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold, \RepeatedKFold, GroupKFoldfrom sklearn.utils import shuffleimport pandas as pd-import os-import contextlibfrom .ml import compute_estimator, train_estimator, get_estimator_class, \get_classification_objective@@ -56,6 +54,7 @@ class SearchState:self.low_cost_partial_config = {}self.cat_hp_cost = {}self.data_size = data_size+ self.ls_ever_converged = Falsesearch_space = learner_class.search_space(data_size=data_size, task=task)for name, space in search_space.items():@@ -215,7 +214,6 @@ class AutoMLState:}if sampled_weight is not None:self.fit_kwargs['sample_weight'] = weight- # with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):# tune.report(**result)return result@@ -820,10 +818,12 @@ class AutoML:Returns:A dict.(a) if there is only one estimator in estimator_list, each key is a- hyperparameter name+ hyperparameter name.(b) otherwise, it is a nested dict with 'ml' as the key, anda list of the low_cost_partial_configs as the value, corresponding- to each learner's low_cost_partial_config+ to each learner's low_cost_partial_config; the estimator index as+ an integer corresponding to the cheapest learner is appeneded to the+ list at the end.'''if len(self.estimator_list) == 1:@@ -835,6 +835,9 @@ class AutoML:for estimator in self.estimator_list:c = self._search_states[estimator].low_cost_partial_configconfigs.append(c)+ configs.append(np.argmin([+ self._state.learner_classes.get(estimator).cost_relative2lgbm()+ for estimator in self.estimator_list]))config = {'ml': configs}return config@@ -845,10 +848,11 @@ class AutoML:Returns:A dict.(a) if there is only one estimator in estimator_list, each key is a- hyperparameter name+ hyperparameter name.(b) otherwise, it is a nested dict with 'ml' as the key, anda list of the cat_hp_cost's as the value, corresponding- to each learner's cat_hp_cost+ to each learner's cat_hp_cost; the cost relative to lgbm for each+ learner (as a list itself) is appended to the list at the end.'''if len(self.estimator_list) == 1:@@ -860,6 +864,9 @@ class AutoML:for estimator in self.estimator_list:c = self._search_states[estimator].cat_hp_costconfigs.append(c)+ configs.append([+ self._state.learner_classes.get(estimator).cost_relative2lgbm()+ for estimator in self.estimator_list])config = {'ml': configs}return config@@ -930,7 +937,8 @@ class AutoML:config['FLAML_sample_size'] = sample_sizeestimator = config['learner']del config['learner']- states[estimator].training_function(config)+ result = states[estimator].training_function(config)+ return resultreturn train@@ -943,7 +951,7 @@ class AutoML:'''def size_func(config: dict) -> float:- config = config.get('ml', config).copy+ config = config.get('ml', config)estimator = config['learner']learner_class = self._state.learner_classes.get(estimator)return learner_class.size(config)@@ -971,7 +979,7 @@ class AutoML:metric='auto',task='classification',n_jobs=-1,- log_file_name='default.log',+ log_file_name='flaml.log',estimator_list='auto',time_budget=60,max_iter=1000000,@@ -996,6 +1004,7 @@ class AutoML:learner_selector='sample',hpo_method=None,starting_points={},+ seed=None,**fit_kwargs):'''Find a model for a given task@@ -1063,12 +1072,20 @@ class AutoML:samples used while splitting the dataset into train/valid setverbose: int, default=1 | Controls the verbosity, higher means moremessages.+ hpo_method: str or None, default=None | The hyperparameter+ optimization method. When it is None, CFO is used.+ No need to set when using flaml's default search space or using+ a simple customized search space. When set to 'bs', BlendSearch+ is used. BlendSearch can be tried when the search space is+ complex, for example, containing multiple disjoint, discontinuous+ subspaces.starting_points: A dictionary to specify the starting hyperparameterconfig for the estimators.Keys are the name of the estimators, and values are the startinghyperparamter configurations for the corresponding estimators.+ seed: int or None, default=None | The random seed for np.random.**fit_kwargs: Other key word arguments to pass to fit() function of- the searched learners, such sample_weight+ the searched learners, such as sample_weight.'''self._start_time_flag = time.time()self._state.task = task@@ -1079,6 +1096,8 @@ class AutoML:self._validate_data(X_train, y_train, dataframe, label, X_val, y_val)self._search_states = {} # key: estimator name; value: SearchStateself._random = np.random.RandomState(RANDOM_SEED)+ if seed is not None:+ np.random.seed(seed)self._learner_selector = learner_selectorold_level = logger.getEffectiveLevel()self.verbose = verbose@@ -1144,21 +1163,33 @@ class AutoML:estimator_list))self.estimator_list = estimator_listself._hpo_method = hpo_method or 'cfo'- with training_log_writer(log_file_name) as save_helper:- self._training_log = save_helper- self._state.time_budget = time_budget- self._active_estimators = estimator_list.copy()- self._ensemble = ensemble- self._max_iter = max_iter- self._mem_thres = mem_thres- self._pred_time_limit = pred_time_limit- self._state.train_time_limit = train_time_limit- self._log_type = log_type- self.split_ratio = split_ratio- self._save_model_history = model_history- self._state.n_jobs = n_jobs+ self._state.time_budget = time_budget+ self._active_estimators = estimator_list.copy()+ self._ensemble = ensemble+ self._max_iter = max_iter+ self._mem_thres = mem_thres+ self._pred_time_limit = pred_time_limit+ self._state.train_time_limit = train_time_limit+ self._log_type = log_type+ self.split_ratio = split_ratio+ self._save_model_history = model_history+ self._state.n_jobs = n_jobs+ if log_file_name:+ with training_log_writer(log_file_name) as save_helper:+ self._training_log = save_helper+ self._search()+ else:+ self._training_log = Noneself._search()- logger.info("fit succeeded")+ logger.info("fit succeeded")+ logger.info(f"Time taken to find the best model: {self._time_taken_best_iter}")+ if self._time_taken_best_iter >= time_budget * 0.7 and not \+ all(self._ever_converged_per_learner.values()):+ logger.warn("Time taken to find the best model is {0:.0f}% of the "+ "provided time budget and not all estimators' hyperparameter "+ "search converged. Consider increasing the time budget.".format(+ self._time_taken_best_iter / time_budget * 100))+if verbose == 0:logger.setLevel(old_level)@@ -1169,14 +1200,18 @@ class AutoML:self._state.time_from_start = 0self._estimator_index = Noneself._best_iteration = 0+ self._time_taken_best_iter = 0self._model_history = {}self._config_history = {}self._max_iter_per_learner = 1000000 # TODOself._iter_per_learner = dict([(e, 0) for e in self.estimator_list])+ self._ever_converged_per_learner = dict([(e, False) for e in self.estimator_list])self._fullsize_reached = Falseself._trained_estimator = Noneself._best_estimator = Noneself._retrained_config = {}+ self._warn_threshold = 10+est_retrain_time = next_trial_time = 0best_config_sig = None# use ConcurrencyLimiter to limit the amount of concurrency when@@ -1185,20 +1220,27 @@ class AutoML:if self._ensemble:self.best_model = {}try:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.suggest import ConcurrencyLimiter- except ImportError:+ except (ImportError, AssertionError):from .searcher.suggestion import ConcurrencyLimiterif self._hpo_method in ('cfo', 'grid'):from flaml import CFO as SearchAlgoelif 'optuna' == self._hpo_method:try:+ assert ray_version >= '1.0.0'from ray.tune.suggest.optuna import OptunaSearch as SearchAlgo- except ImportError:+ except (ImportError, AssertionError):from .searcher.suggestion import OptunaSearch as SearchAlgoelif 'bs' == self._hpo_method:from flaml import BlendSearch as SearchAlgo+ elif 'cfocat' == self._hpo_method:+ from flaml import CFOCat as SearchAlgoelse:- raise NotImplementedError+ raise NotImplementedError(+ f"hpo_method={self._hpo_method} is not recognized. "+ "'cfo' and 'bs' are supported.")for self._track_iter in range(self._max_iter):if self._estimator_index is None:@@ -1244,7 +1286,7 @@ class AutoML:else:points_to_evaluate = [search_state.init_config]low_cost_partial_config = search_state.low_cost_partial_config- if self._hpo_method in ('bs', 'cfo', 'grid'):+ if self._hpo_method in ('bs', 'cfo', 'grid', 'cfocat'):algo = SearchAlgo(metric='val_loss', mode='min', space=search_space,points_to_evaluate=points_to_evaluate,@@ -1267,7 +1309,7 @@ class AutoML:max_concurrent=1)else:search_space = None- if self._hpo_method in ('bs', 'cfo'):+ if self._hpo_method in ('bs', 'cfo', 'cfocat'):search_state.search_alg.set_search_properties(metric=None, mode=None,config={@@ -1320,20 +1362,22 @@ class AutoML:self._trained_estimator = Noneself._trained_estimator = search_state.trained_estimatorself._best_iteration = self._track_iter+ self._time_taken_best_iter = self._state.time_from_startbetter = Truenext_trial_time = search_state.time2eval_bestif better or self._log_type == 'all':- self._training_log.append(- self._iter_per_learner[estimator],- search_state.train_loss,- search_state.trial_time,- self._state.time_from_start,- search_state.val_loss,- search_state.config,- search_state.best_loss,- search_state.best_config,- estimator,- search_state.sample_size)+ if self._training_log:+ self._training_log.append(+ self._iter_per_learner[estimator],+ search_state.train_loss,+ search_state.trial_time,+ self._state.time_from_start,+ search_state.val_loss,+ search_state.config,+ search_state.best_loss,+ search_state.best_config,+ estimator,+ search_state.sample_size)if mlflow is not None and mlflow.active_run():with mlflow.start_run(nested=True):mlflow.log_metric('iter_counter',@@ -1365,6 +1409,15 @@ class AutoML:search_state.best_loss,self._best_estimator,self._state.best_loss))+ searcher = search_state.search_alg.searcher+ if searcher.is_ls_ever_converged and not self._ever_converged_per_learner[estimator]:+ self._ever_converged_per_learner[estimator] = searcher.is_ls_ever_converged+ if all(self._ever_converged_per_learner.values()) and \+ self._state.time_from_start > self._warn_threshold * self._time_taken_best_iter:+ logger.warn("All estimator hyperparameters local search has converged at least once, "+ f"and the total search time exceeds {self._warn_threshold} times the time taken "+ "to find the best model.")+ self._warn_threshold *= 10else:logger.info(f"no enough budget for learner {estimator}")if self._estimator_index is not None:@@ -1396,7 +1449,8 @@ class AutoML:if time_left < time_ensemble < 2 * time_left:break# Add a checkpoint for the current best config to the log.- self._training_log.checkpoint()+ if self._training_log:+ self._training_log.checkpoint()if self._best_estimator:self._selected = self._search_states[self._best_estimator]self._trained_estimator = self._selected.trained_estimatordiff --git a/flaml/data.py b/flaml/data.pyindex 06d45ee37..c552ffeeb 100644--- a/flaml/data.py+++ b/flaml/data.py@@ -11,7 +11,8 @@ from .training_log import training_log_readerfrom datetime import datetime-def load_openml_dataset(dataset_id, data_dir=None, random_state=0):+def load_openml_dataset(dataset_id, data_dir=None, random_state=0,+ dataset_format='dataframe'):'''Load dataset from open ML.If the file is not cached locally, download it from open ML.@@ -20,12 +21,15 @@ def load_openml_dataset(dataset_id, data_dir=None, random_state=0):dataset_id: An integer of the dataset id in openmldata_dir: A string of the path to store and load the datarandom_state: An integer of the random seed for splitting data-+ dataset_format: A string specifying the format of returned dataset. Default is 'dataframe'.+ Can choose from ['dataframe', 'array'].+ If 'dataframe', the returned dataset will be a Pandas DataFrame.+ If 'array', the returned dataset will be a NumPy array or a SciPy sparse matrix.Returns:- X_train: A dataframe of training data- X_test: A dataframe of test data- y_train: A series of labels for training data- y_test: A series of labels for test data+ X_train: Training data+ X_test: Test data+ y_train: A series or array of labels for training data+ y_test: A series or array of labels for test data'''import osimport openml@@ -48,7 +52,7 @@ def load_openml_dataset(dataset_id, data_dir=None, random_state=0):print('Dataset name:', dataset.name)X, y, * \__ = dataset.get_data(- target=dataset.default_target_attribute)+ target=dataset.default_target_attribute, dataset_format=dataset_format)X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state)print(diff --git a/flaml/ml.py b/flaml/ml.pyindex 8847f407c..3fc0bbf73 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -23,24 +23,24 @@ logger = logging.getLogger(__name__)def get_estimator_class(task, estimator_name):''' when adding a new learner, need to add an elif branch '''- if 'xgboost' in estimator_name:+ if 'xgboost' == estimator_name:if 'regression' in task:estimator_class = XGBoostEstimatorelse:estimator_class = XGBoostSklearnEstimator- elif 'rf' in estimator_name:+ elif 'rf' == estimator_name:estimator_class = RandomForestEstimator- elif 'lgbm' in estimator_name:+ elif 'lgbm' == estimator_name:estimator_class = LGBMEstimator- elif 'lrl1' in estimator_name:+ elif 'lrl1' == estimator_name:estimator_class = LRL1Classifier- elif 'lrl2' in estimator_name:+ elif 'lrl2' == estimator_name:estimator_class = LRL2Classifier- elif 'catboost' in estimator_name:+ elif 'catboost' == estimator_name:estimator_class = CatBoostEstimator- elif 'extra_tree' in estimator_name:+ elif 'extra_tree' == estimator_name:estimator_class = ExtraTreeEstimator- elif 'kneighbor' in estimator_name:+ elif 'kneighbor' == estimator_name:estimator_class = KNeighborsEstimatorelse:raise ValueError(diff --git a/flaml/model.py b/flaml/model.pyindex fa88c8e81..341714085 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -183,9 +183,10 @@ class SKLearnEstimator(BaseEstimator):def _preprocess(self, X):if isinstance(X, pd.DataFrame):- X = X.copy()cat_columns = X.select_dtypes(include=['category']).columns- X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)+ if not cat_columns.empty:+ X = X.copy()+ X[cat_columns] = X[cat_columns].apply(lambda x: x.cat.codes)elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif':# numpy array is not of numeric dtypeX = pd.DataFrame(X)@@ -213,7 +214,7 @@ class LGBMEstimator(BaseEstimator):'low_cost_init_value': 4,},'min_child_samples': {- 'domain': tune.lograndint(lower=2, upper=2**7),+ 'domain': tune.lograndint(lower=2, upper=2**7 + 1),'init_value': 20,},'learning_rate': {@@ -225,7 +226,7 @@ class LGBMEstimator(BaseEstimator):'init_value': 1.0,},'log_max_bin': {- 'domain': tune.lograndint(lower=3, upper=10),+ 'domain': tune.lograndint(lower=3, upper=11),'init_value': 8,},'colsample_bytree': {@@ -270,6 +271,8 @@ class LGBMEstimator(BaseEstimator):self.params["objective"] = objectiveif "max_bin" not in self.params:self.params['max_bin'] = 1 << int(round(log_max_bin)) - 1+ if "verbose" not in self.params:+ self.params['verbose'] = -1if 'regression' in task:self.estimator_class = LGBMRegressorelse:@@ -281,6 +284,13 @@ class LGBMEstimator(BaseEstimator):if not isinstance(X, pd.DataFrame) and issparse(X) and np.issubdtype(X.dtype, np.integer):X = X.astype(float)+ elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif':+ # numpy array is not of numeric dtype+ X = pd.DataFrame(X)+ for col in X.columns:+ if isinstance(X[col][0], str):+ X[col] = X[col].astype('category').cat.codes+ X = X.to_numpy()return Xdef fit(self, X_train, y_train, budget=None, **kwargs):@@ -455,14 +465,15 @@ class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):super().__init__(task, **params)del self.params['objective']del self.params['max_bin']+ del self.params['verbose']self.params.update({"n_estimators": int(round(n_estimators)),'max_leaves': int(round(max_leaves)),'max_depth': 0,'grow_policy': params.get("grow_policy", 'lossguide'),'tree_method': tree_method,- 'verbosity': 0,'n_jobs': n_jobs,+ 'verbosity': 0,'learning_rate': float(learning_rate),'subsample': float(subsample),'reg_alpha': float(reg_alpha),@@ -531,6 +542,7 @@ class RandomForestEstimator(SKLearnEstimator, LGBMEstimator):self.params.update({"n_estimators": int(round(n_estimators)),"n_jobs": n_jobs,+ "verbose": 0,'max_features': float(max_features),"max_leaf_nodes": params.get('max_leaf_nodes', int(round(max_leaves))),})@@ -629,7 +641,7 @@ class CatBoostEstimator(BaseEstimator):@classmethoddef search_space(cls, data_size, **params):- upper = max(min(round(1500000 / data_size), 150), 11)+ upper = max(min(round(1500000 / data_size), 150), 12)return {'early_stopping_rounds': {'domain': tune.lograndint(lower=10, upper=upper),@@ -657,6 +669,25 @@ class CatBoostEstimator(BaseEstimator):CatBoostEstimator._time_per_iter = NoneCatBoostEstimator._train_size = 0+ def _preprocess(self, X):+ if isinstance(X, pd.DataFrame):+ cat_columns = X.select_dtypes(include=['category']).columns+ if not cat_columns.empty:+ X = X.copy()+ X[cat_columns] = X[cat_columns].apply(+ lambda x:+ x.cat.rename_categories(+ [str(c) if isinstance(c, float) else c+ for c in x.cat.categories]))+ elif isinstance(X, np.ndarray) and X.dtype.kind not in 'buif':+ # numpy array is not of numeric dtype+ X = pd.DataFrame(X)+ for col in X.columns:+ if isinstance(X[col][0], str):+ X[col] = X[col].astype('category').cat.codes+ X = X.to_numpy()+ return X+def __init__(self, task='binary:logistic', n_jobs=1,n_estimators=8192, learning_rate=0.1, early_stopping_rounds=4, **params@@ -685,68 +716,69 @@ class CatBoostEstimator(BaseEstimator):def fit(self, X_train, y_train, budget=None, **kwargs):start_time = time.time()n_iter = self.params["n_estimators"]+ X_train = self._preprocess(X_train)if isinstance(X_train, pd.DataFrame):cat_features = list(X_train.select_dtypes(include='category').columns)else:cat_features = []- from catboost import CatBoostError- try:- if (not CatBoostEstimator._time_per_iter or abs(- CatBoostEstimator._train_size - len(y_train)) > 4) and budget:- # measure the time per iteration- self.params["n_estimators"] = 1- CatBoostEstimator._smallmodel = self.estimator_class(**self.params)- CatBoostEstimator._smallmodel.fit(- X_train, y_train, cat_features=cat_features, **kwargs)- CatBoostEstimator._t1 = time.time() - start_time- if CatBoostEstimator._t1 >= budget:- self.params["n_estimators"] = n_iter- self._model = CatBoostEstimator._smallmodel- return CatBoostEstimator._t1- self.params["n_estimators"] = 4- CatBoostEstimator._smallmodel = self.estimator_class(**self.params)- CatBoostEstimator._smallmodel.fit(- X_train, y_train, cat_features=cat_features, **kwargs)- CatBoostEstimator._time_per_iter = (- time.time() - start_time - CatBoostEstimator._t1) / (- self.params["n_estimators"] - 1)- if CatBoostEstimator._time_per_iter <= 0:- CatBoostEstimator._time_per_iter = CatBoostEstimator._t1- CatBoostEstimator._train_size = len(y_train)- if time.time() - start_time >= budget or n_iter == self.params[- "n_estimators"]:- self.params["n_estimators"] = n_iter- self._model = CatBoostEstimator._smallmodel- return time.time() - start_time- if budget:- train_times = 1- self.params["n_estimators"] = min(n_iter, int(- (budget - time.time() + start_time - CatBoostEstimator._t1)- / train_times / CatBoostEstimator._time_per_iter + 1))+ # from catboost import CatBoostError+ # try:+ if (not CatBoostEstimator._time_per_iter or abs(+ CatBoostEstimator._train_size - len(y_train)) > 4) and budget:+ # measure the time per iteration+ self.params["n_estimators"] = 1+ CatBoostEstimator._smallmodel = self.estimator_class(**self.params)+ CatBoostEstimator._smallmodel.fit(+ X_train, y_train, cat_features=cat_features, **kwargs)+ CatBoostEstimator._t1 = time.time() - start_time+ if CatBoostEstimator._t1 >= budget:+ self.params["n_estimators"] = n_iterself._model = CatBoostEstimator._smallmodel- if self.params["n_estimators"] > 0:- n = max(int(len(y_train) * 0.9), len(y_train) - 1000)- X_tr, y_tr = X_train[:n], y_train[:n]- if 'sample_weight' in kwargs:- weight = kwargs['sample_weight']- if weight is not None:- kwargs['sample_weight'] = weight[:n]- else:- weight = None- from catboost import Pool- model = self.estimator_class(**self.params)- model.fit(- X_tr, y_tr, cat_features=cat_features,- eval_set=Pool(- data=X_train[n:], label=y_train[n:],- cat_features=cat_features),- **kwargs) # model.get_best_iteration()+ return CatBoostEstimator._t1+ self.params["n_estimators"] = 4+ CatBoostEstimator._smallmodel = self.estimator_class(**self.params)+ CatBoostEstimator._smallmodel.fit(+ X_train, y_train, cat_features=cat_features, **kwargs)+ CatBoostEstimator._time_per_iter = (+ time.time() - start_time - CatBoostEstimator._t1) / (+ self.params["n_estimators"] - 1)+ if CatBoostEstimator._time_per_iter <= 0:+ CatBoostEstimator._time_per_iter = CatBoostEstimator._t1+ CatBoostEstimator._train_size = len(y_train)+ if time.time() - start_time >= budget or n_iter == self.params[+ "n_estimators"]:+ self.params["n_estimators"] = n_iter+ self._model = CatBoostEstimator._smallmodel+ return time.time() - start_time+ if budget:+ train_times = 1+ self.params["n_estimators"] = min(n_iter, int(+ (budget - time.time() + start_time - CatBoostEstimator._t1)+ / train_times / CatBoostEstimator._time_per_iter + 1))+ self._model = CatBoostEstimator._smallmodel+ if self.params["n_estimators"] > 0:+ n = max(int(len(y_train) * 0.9), len(y_train) - 1000)+ X_tr, y_tr = X_train[:n], y_train[:n]+ if 'sample_weight' in kwargs:+ weight = kwargs['sample_weight']if weight is not None:- kwargs['sample_weight'] = weight- self._model = model- except CatBoostError:- self._model = None+ kwargs['sample_weight'] = weight[:n]+ else:+ weight = None+ from catboost import Pool+ model = self.estimator_class(**self.params)+ model.fit(+ X_tr, y_tr, cat_features=cat_features,+ eval_set=Pool(+ data=X_train[n:], label=y_train[n:],+ cat_features=cat_features),+ **kwargs) # model.get_best_iteration()+ if weight is not None:+ kwargs['sample_weight'] = weight+ self._model = model+ # except CatBoostError:+ # self._model = Noneself.params["n_estimators"] = n_itertrain_time = time.time() - start_timereturn train_timediff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 88884f982..43f6b3362 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -9,16 +9,18 @@ import timeimport pickletry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.suggest import Searcherfrom ray.tune.suggest.optuna import OptunaSearch as GlobalSearch- from ray.tune.suggest.variant_generator import generate_variantsfrom ray.tune.utils.util import flatten_dict-except ImportError:+except (ImportError, AssertionError):from .suggestion import Searcherfrom .suggestion import OptunaSearch as GlobalSearch- from .variant_generator import generate_variants, flatten_dict+ from .variant_generator import flatten_dictfrom .search_thread import SearchThreadfrom .flow2 import FLOW2+from ..tune.space import add_cost_to_space, normalize # TODO: , define_by_run_funcimport logginglogger = logging.getLogger(__name__)@@ -125,9 +127,15 @@ class BlendSearch(Searcher):if self._metric_constraints:# metric modified by lagrangemetric += self.lagrange+ self._cat_hp_cost = cat_hp_cost or {}+ if space:+ add_cost_to_space(space, init_config, self._cat_hp_cost)if global_search_alg is not None:self._gs = global_search_algelif getattr(self, '__name__', None) != 'CFO':+ gs_space = space+ # TODO: when define_by_run is supported+ # gs_space = define_by_run_func(space)try:gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)if experimental:@@ -137,10 +145,11 @@ class BlendSearch(Searcher):else:sampler = Noneself._gs = GlobalSearch(- space=space, metric=metric, mode=mode, seed=gs_seed,+ space=gs_space, metric=metric, mode=mode, seed=gs_seed,sampler=sampler)except TypeError:- self._gs = GlobalSearch(space=space, metric=metric, mode=mode)+ self._gs = GlobalSearch(space=gs_space, metric=metric, mode=mode)+ self._gs.space = spaceelse:self._gs = Noneself._experimental = experimental@@ -152,8 +161,10 @@ class BlendSearch(Searcher):else:self._candidate_start_points = Noneself._ls = self.LocalSearch(- init_config, metric, mode, cat_hp_cost, space, prune_attr,+ init_config, metric, mode, space, prune_attr,min_resource, max_resource, reduction_factor, self.cost_attr, seed)+ self._is_ls_ever_converged = False+ self._subspace = {} # the subspace for each trial idself._init_search()def set_search_properties(self,@@ -174,9 +185,13 @@ class BlendSearch(Searcher):self._mode = modeif not self._ls.space:# the search space can be set only once- self._ls.set_search_properties(metric, mode, config)if self._gs is not None:self._gs.set_search_properties(metric, mode, config)+ self._gs.space = config+ if config:+ add_cost_to_space(+ config, self._ls.init_config, self._cat_hp_cost)+ self._ls.set_search_properties(metric, mode, config)self._init_search()elif metric_changed or mode_changed:# reset search when metric or mode changed@@ -205,7 +220,9 @@ class BlendSearch(Searcher):self._thread_count = 1 # total # threads createdself._init_used = self._ls.init_config is Noneself._trial_proposed_by = {} # trial_id: str -> thread_id: int- self._ls_bound_min = self._ls.normalize(self._ls.init_config)+ self._ls_bound_min = normalize(+ self._ls.init_config.copy(), self._ls.space, self._ls.init_config,+ {}, recursive=True)self._ls_bound_max = self._ls_bound_min.copy()self._gs_admissible_min = self._ls_bound_min.copy()self._gs_admissible_max = self._ls_bound_max.copy()@@ -231,34 +248,16 @@ class BlendSearch(Searcher):'''with open(checkpoint_path, "rb") as inputFile:state = pickle.load(inputFile)- self._metric_target = state._metric_target- self._search_thread_pool = state._search_thread_pool- self._thread_count = state._thread_count- self._init_used = state._init_used- self._trial_proposed_by = state._trial_proposed_by- self._ls_bound_min = state._ls_bound_min- self._ls_bound_max = state._ls_bound_max- self._gs_admissible_min = state._gs_admissible_min- self._gs_admissible_max = state._gs_admissible_max- self._result = state._result- self._deadline = state._deadline- self._metric, self._mode = state._metric, state._mode- self._points_to_evaluate = state._points_to_evaluate- self._gs = state._gs- self._ls = state._ls- self._config_constraints = state._config_constraints- self._metric_constraints = state._metric_constraints- self._metric_constraint_satisfied = state._metric_constraint_satisfied- self._metric_constraint_penalty = state._metric_constraint_penalty- self._candidate_start_points = state._candidate_start_points- if self._candidate_start_points:- self._started_from_given = state._started_from_given- self._started_from_low_cost = state._started_from_low_cost+ self.__dict__ = state.__dict__@propertydef metric_target(self):return self._metric_target+ @property+ def is_ls_ever_converged(self):+ return self._is_ls_ever_converged+def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None,error: bool = False):''' search thread updater and cleaner@@ -296,10 +295,12 @@ class BlendSearch(Searcher):for key, value in result.items():if key.startswith('config/'):config[key[7:]] = value+ signature = self._ls.config_signature(+ config, self._subspace.get(trial_id, {}))if error: # remove from result cache- del self._result[self._ls.config_signature(config)]+ del self._result[signature]else: # add to result cache- self._result[self._ls.config_signature(config)] = result+ self._result[signature] = result# update target metric if improvedobjective = result[self._ls.metric]if (objective - self._metric_target) * self._ls.metric_op < 0:@@ -307,8 +308,11 @@ class BlendSearch(Searcher):if thread_id:if not self._metric_constraint_satisfied:# no point has been found to satisfy metric constraint- self._expand_admissible_region()+ self._expand_admissible_region(+ self._ls_bound_min, self._ls_bound_max,+ self._subspace.get(trial_id, self._ls.space))if self._gs is not None and self._experimental:+ # TODO: key match for hierarchical spaceself._gs.add_evaluated_point(flatten_dict(config), objective)elif metric_constraint_satisfied and self._create_condition(result):@@ -320,7 +324,8 @@ class BlendSearch(Searcher):del self._candidate_start_points[trial_id]else:self._started_from_low_cost = True- self._create_thread(config, result)+ self._create_thread(config, result, self._subspace.get(+ trial_id, self._ls.space))# reset admissible region to ls bounding boxself._gs_admissible_min.update(self._ls_bound_min)self._gs_admissible_max.update(self._ls_bound_max)@@ -328,29 +333,46 @@ class BlendSearch(Searcher):if thread_id and thread_id in self._search_thread_pool:# local search threadself._clean(thread_id)+ if trial_id in self._subspace and not (self._candidate_start_points+ and trial_id in self._candidate_start_points):+ del self._subspace[trial_id]- def _create_thread(self, config, result):+ def _create_thread(self, config, result, space):# logger.info(f"create local search thread from {config}")self._search_thread_pool[self._thread_count] = SearchThread(self._ls.mode,self._ls.create(config, result[self._ls.metric],- cost=result.get(self.cost_attr, 1)),+ cost=result.get(self.cost_attr, 1), space=space),self.cost_attr)self._thread_count += 1self._update_admissible_region(- config, self._ls_bound_min, self._ls_bound_max)+ config, self._ls_bound_min, self._ls_bound_max, space)- def _update_admissible_region(self, config, admissible_min, admissible_max):+ def _update_admissible_region(+ self, config, admissible_min, admissible_max, space: Dict = {}+ ):# update admissible region- normalized_config = self._ls.normalize(config)+ normalized_config = normalize(config, space, config, {})for key in admissible_min:value = normalized_config[key]- if value > admissible_max[key]:- admissible_max[key] = value- elif value < admissible_min[key]:- admissible_min[key] = value+ if isinstance(admissible_max[key], list):+ choice = space[key]['_choice_']+ self._update_admissible_region(+ value,+ admissible_min[key][choice], admissible_max[key][choice],+ space[key]+ )+ elif isinstance(value, dict):+ self._update_admissible_region(+ value,+ admissible_min[key], admissible_max[key], space[key])+ else:+ if value > admissible_max[key]:+ admissible_max[key] = value+ elif value < admissible_min[key]:+ admissible_min[key] = valuedef _create_condition(self, result: Dict) -> bool:''' create thread condition@@ -379,8 +401,11 @@ class BlendSearch(Searcher):breakcreate_new = Falseif self._search_thread_pool[thread_id].converged:+ self._is_ls_ever_converged = Truetodelete.add(thread_id)- self._expand_admissible_region()+ self._expand_admissible_region(+ self._ls_bound_min, self._ls_bound_max,+ self._search_thread_pool[thread_id].space)if self._candidate_start_points:if not self._started_from_given:# remove start points whose perf is worse than the converged@@ -417,12 +442,21 @@ class BlendSearch(Searcher):config[key[7:]] = valueself._started_from_given = Truedel self._candidate_start_points[best_trial_id]- self._create_thread(config, result)-- def _expand_admissible_region(self):- for key in self._ls_bound_max:- self._ls_bound_max[key] += self._ls.STEPSIZE- self._ls_bound_min[key] -= self._ls.STEPSIZE+ self._create_thread(config, result, self._subspace.get(+ best_trial_id, self._ls.space))++ def _expand_admissible_region(self, lower, upper, space):+ for key in upper:+ ub = upper[key]+ if isinstance(ub, list):+ choice = space[key]['_choice_']+ self._expand_admissible_region(+ lower[key][choice], upper[key][choice], space[key])+ elif isinstance(ub, dict):+ self._expand_admissible_region(lower[key], ub, space[key])+ else:+ upper[key] += self._ls.STEPSIZE+ lower[key] -= self._ls.STEPSIZEdef _inferior(self, id1: int, id2: int) -> bool:''' whether thread id1 is inferior to id2@@ -460,34 +494,42 @@ class BlendSearch(Searcher):if choice and config is None:# local search thread finishesif self._search_thread_pool[choice].converged:- self._expand_admissible_region()+ self._expand_admissible_region(+ self._ls_bound_min, self._ls_bound_max,+ self._search_thread_pool[choice].space)del self._search_thread_pool[choice]return None# preliminary check; not checking config validation- skip = self._should_skip(choice, trial_id, config)+ space = self._search_thread_pool[choice].space+ skip = self._should_skip(choice, trial_id, config, space)+ use_rs = 0if skip:if choice:return None# use rs when BO fails to suggest a config- for _, generated in generate_variants({'config': self._ls.space}):- config = generated['config']- break # get one random config- skip = self._should_skip(-1, trial_id, config)+ config, space = self._ls.complete_config({})+ skip = self._should_skip(-1, trial_id, config, space)if skip:return None- if choice or self._valid(config):+ use_rs = 1+ if choice or self._valid(+ config, space, self._gs_admissible_min, self._gs_admissible_max):# LS or valid or no backup choiceself._trial_proposed_by[trial_id] = choice+ self._search_thread_pool[choice].running += use_rselse: # invalid config proposed by GSif choice == backup:# use CFO's init pointinit_config = self._ls.init_config- config = self._ls.complete_config(+ config, space = self._ls.complete_config(init_config, self._ls_bound_min, self._ls_bound_max)self._trial_proposed_by[trial_id] = choice+ self._search_thread_pool[choice].running += 1else:- config = self._search_thread_pool[backup].suggest(trial_id)- skip = self._should_skip(backup, trial_id, config)+ thread = self._search_thread_pool[backup]+ config = thread.suggest(trial_id)+ space = thread.space+ skip = self._should_skip(backup, trial_id, config, space)if skip:return Noneself._trial_proposed_by[trial_id] = backup@@ -498,21 +540,24 @@ class BlendSearch(Searcher):config[self._ls.prune_attr] = self._ls.min_resource# temporarily relax admissible region for parallel proposalsself._update_admissible_region(- config, self._gs_admissible_min, self._gs_admissible_max)+ config, self._gs_admissible_min, self._gs_admissible_max,+ space)else:self._update_admissible_region(- config, self._ls_bound_min, self._ls_bound_max)+ config, self._ls_bound_min, self._ls_bound_max, space)self._gs_admissible_min.update(self._ls_bound_min)self._gs_admissible_max.update(self._ls_bound_max)- self._result[self._ls.config_signature(config)] = {}+ signature = self._ls.config_signature(config, space)+ self._result[signature] = {}+ self._subspace[trial_id] = spaceelse: # use init configif self._candidate_start_points is not None and self._points_to_evaluate:self._candidate_start_points[trial_id] = Noneinit_config = self._points_to_evaluate.pop(0) if self._points_to_evaluate else self._ls.init_config- config = self._ls.complete_config(+ config, space = self._ls.complete_config(init_config, self._ls_bound_min, self._ls_bound_max)- config_signature = self._ls.config_signature(config)+ config_signature = self._ls.config_signature(config, space)result = self._result.get(config_signature)if result: # tried beforereturn None@@ -523,15 +568,16 @@ class BlendSearch(Searcher):self._init_used = Trueself._trial_proposed_by[trial_id] = 0self._search_thread_pool[0].running += 1+ self._subspace[trial_id] = spacereturn config- def _should_skip(self, choice, trial_id, config) -> bool:+ def _should_skip(self, choice, trial_id, config, space) -> bool:''' if config is None or config's result is known or constraints are violatedreturn True; o.w. return False'''if config is None:return True- config_signature = self._ls.config_signature(config)+ config_signature = self._ls.config_signature(config, space)exists = config_signature in self._result# check constraintsif not exists and self._config_constraints:@@ -600,23 +646,35 @@ class BlendSearch(Searcher):backup_thread_id = thread_idreturn top_thread_id, backup_thread_id- def _valid(self, config: Dict) -> bool:+ def _valid(self, config: Dict, space: Dict, lower: Dict, upper: Dict) -> bool:''' config validator'''- normalized_config = self._ls.normalize(config)- for key in self._gs_admissible_min:+ normalized_config = normalize(config, space, config, {})+ for key, lb in lower.items():if key in config:value = normalized_config[key]- if value + self._ls.STEPSIZE < self._gs_admissible_min[key] \- or value > self._gs_admissible_max[key] + self._ls.STEPSIZE:+ if isinstance(lb, list):+ subspace = space[key]['_choice_']+ elif isinstance(lb, dict):+ subspace = space[key]+ else:+ subspace = None+ if subspace:+ valid = self._valid(value, subspace, lb, upper[key])+ if not valid:+ return False+ elif (value + self._ls.STEPSIZE < lower[key]+ or value > upper[key] + self._ls.STEPSIZE):return Falsereturn Truetry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune import (uniform, quniform, choice, randint, qrandint, randn,qrandn, loguniform, qloguniform)-except ImportError:+except (ImportError, AssertionError):from ..tune.sample import (uniform, quniform, choice, randint, qrandint, randn,qrandn, loguniform, qloguniform)diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.pyindex b807e6fe2..5ad75d89f 100644--- a/flaml/searcher/flow2.py+++ b/flaml/searcher/flow2.py@@ -3,17 +3,20 @@* Licensed under the MIT License. See LICENSE file in the* project root for license information.'''-from typing import Dict, Optional+from typing import Dict, Optional, Tupleimport numpy as nptry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.suggest import Searcherfrom ray.tune.suggest.variant_generator import generate_variantsfrom ray.tune import samplefrom ray.tune.utils.util import flatten_dict, unflatten_dict-except ImportError:+except (ImportError, AssertionError):from .suggestion import Searcherfrom .variant_generator import generate_variants, flatten_dict, unflatten_dictfrom ..tune import sample+from ..tune.space import complete_config, denormalize, normalizeimport logging@@ -31,7 +34,6 @@ class FLOW2(Searcher):init_config: dict,metric: Optional[str] = None,mode: Optional[str] = None,- cat_hp_cost: Optional[dict] = None,space: Optional[dict] = None,prune_attr: Optional[str] = None,min_resource: Optional[float] = None,@@ -90,12 +92,11 @@ class FLOW2(Searcher):elif mode == "min":self.metric_op = 1.self.space = space or {}- self.space = flatten_dict(self.space, prevent_delimiter=True)+ self._space = flatten_dict(self.space, prevent_delimiter=True)self._random = np.random.RandomState(seed)self._seed = seedself.init_config = init_configself.best_config = flatten_dict(init_config)- self.cat_hp_cost = cat_hp_costself.prune_attr = prune_attrself.min_resource = min_resourceself.resource_multiple_factor = resource_multiple_factor or 4@@ -109,18 +110,9 @@ class FLOW2(Searcher):def _init_search(self):self._tunable_keys = []self._bounded_keys = []- # choices of numeric values. integer encoding.- # value: (ordered list of choices,- # dict from choice to index in the ordered list)- self._ordered_choice_hp = {}- # choices with given cost. integer encoding.- # value: (array of choices ordered by cost,- # dict from choice to index in the ordered array)- self._ordered_cat_hp = {}- # unordered choices. value: cardinalityself._unordered_cat_hp = {}- self._cat_hp_cost = {}- for key, domain in self.space.items():+ hier = False+ for key, domain in self._space.items():assert not (isinstance(domain, dict) and 'grid_search' in domain), \f"{key}'s domain is grid search, not supported in FLOW^2."if callable(getattr(domain, 'get_sampler', None)):@@ -129,41 +121,33 @@ class FLOW2(Searcher):# the step size lower bound for uniform variables doesn't depend# on the current configif isinstance(sampler, sample.Quantized):- sampler_inner = sampler.get_sampler()- if str(sampler_inner) == 'Uniform':+ q = sampler.q+ sampler = sampler.get_sampler()+ if str(sampler) == 'Uniform':self._step_lb = min(- self._step_lb, sampler.q / (domain.upper - domain.lower))+ self._step_lb, q / (domain.upper - domain.lower))elif isinstance(domain, sample.Integer) and str(sampler) == 'Uniform':self._step_lb = min(- self._step_lb, 1.0 / (domain.upper - domain.lower))+ self._step_lb, 1.0 / (domain.upper - 1 - domain.lower))if isinstance(domain, sample.Categorical):- cat_hp_cost = self.cat_hp_cost- if cat_hp_cost and key in cat_hp_cost:- cost = np.array(cat_hp_cost[key])- ind = np.argsort(cost)- ordered = np.array(domain.categories)[ind]- cost = self._cat_hp_cost[key] = cost[ind]- d = {}- for i, choice in enumerate(ordered):- d[choice] = i- self._ordered_cat_hp[key] = (ordered, d)- elif all(isinstance(x, int) or isinstance(x, float)- for x in domain.categories):- ordered = sorted(domain.categories)- d = {}- for i, choice in enumerate(ordered):- d[choice] = i- self._ordered_choice_hp[key] = (ordered, d)- else:+ if not domain.ordered:self._unordered_cat_hp[key] = len(domain.categories)+ if not hier:+ for cat in domain.categories:+ if isinstance(cat, dict):+ hier = True+ breakif str(sampler) != 'Normal':self._bounded_keys.append(key)- self._space_keys = list(self.space.keys())- if (self.prune_attr and self.prune_attr not in self.space+ if not hier:+ self._space_keys = sorted(self._space.keys())+ self._hierarchical = hier+ if (self.prune_attr and self.prune_attr not in self._spaceand self.max_resource):- self._space_keys.append(self.prune_attr)self.min_resource = self.min_resource or self._min_resource()self._resource = self._round(self.min_resource)+ if not hier:+ self._space_keys.append(self.prune_attr)else:self._resource = Noneself.incumbent = {}@@ -203,20 +187,21 @@ class FLOW2(Searcher):for key in self._tunable_keys:if key not in self.best_config:continue- domain = self.space[key]+ domain = self._space[key]sampler = domain.get_sampler()# the stepsize lower bound for log uniform variables depends on the# current configif isinstance(sampler, sample.Quantized):+ q = sampler.qsampler_inner = sampler.get_sampler()if str(sampler_inner) == 'LogUniform':step_lb = min(- step_lb, np.log(1.0 + sampler.q / self.best_config[key])+ step_lb, np.log(1.0 + q / self.best_config[key])/ np.log(domain.upper / domain.lower))elif isinstance(domain, sample.Integer) and str(sampler) == 'LogUniform':step_lb = min(step_lb, np.log(1.0 + 1.0 / self.best_config[key])- / np.log(domain.upper / domain.lower))+ / np.log((domain.upper - 1) / domain.lower))if np.isinf(step_lb):step_lb = self.STEP_LOWER_BOUNDelse:@@ -246,56 +231,26 @@ class FLOW2(Searcher):def complete_config(self, partial_config: Dict,lower: Optional[Dict] = None, upper: Optional[Dict] = None- ) -> Dict:+ ) -> Tuple[Dict, Dict]:''' generate a complete config from the partial config inputadd minimal resource to config if available'''- if self._reset_times and partial_config == self.init_config:- # not the first time to complete init_config, use random gaussian- normalized = self.normalize(partial_config)- for key in normalized:- # don't change unordered cat choice- if key not in self._unordered_cat_hp:- if upper and lower:- up, low = upper[key], lower[key]- gauss_std = up - low or self.STEPSIZE- # allowed bound- up += self.STEPSIZE- low -= self.STEPSIZE- elif key in self._bounded_keys:- up, low, gauss_std = 1, 0, 1.0- else:- up, low, gauss_std = np.Inf, -np.Inf, 1.0- if key in self._bounded_keys:- up = min(up, 1)- low = max(low, 0)- delta = self.rand_vector_gaussian(1, gauss_std)[0]- normalized[key] = max(low, min(up, normalized[key] + delta))- # use best config for unordered cat choice- config = self.denormalize(normalized)- else:- # first time init_config, or other configs, take as is- config = partial_config.copy()+ disturb = self._reset_times and partial_config == self.init_config+ # if not the first time to complete init_config, use random gaussian+ config, space = complete_config(+ partial_config, self.space, self, disturb, lower, upper)if partial_config == self.init_config:self._reset_times += 1- config = flatten_dict(config)- for key, value in self.space.items():- if key not in config:- config[key] = value- for _, generated in generate_variants({'config': config}):- config = generated['config']- breakif self._resource:config[self.prune_attr] = self.min_resource- return unflatten_dict(config)+ return config, space- def create(self, init_config: Dict, obj: float, cost: float) -> Searcher:- flatten_config = flatten_dict(init_config)- # use the subspace where the init_config is located- space = {k: self.space[k] for k in flatten_config if k in self.space}+ def create(self, init_config: Dict, obj: float, cost: float, space: Dict+ ) -> Searcher:+ # space is the subspace where the init_config is locatedflow2 = self.__class__(- init_config, self.metric, self.mode, self._cat_hp_cost,- unflatten_dict(space), self.prune_attr,+ init_config, self.metric, self.mode,+ space, self.prune_attr,self.min_resource, self.max_resource,self.resource_multiple_factor, self.cost_attr, self._seed + 1)flow2.best_obj = obj * self.metric_op # minimize internally@@ -303,115 +258,17 @@ class FLOW2(Searcher):self._seed += 1return flow2- def normalize(self, config) -> Dict:+ def normalize(self, config, recursive=False) -> Dict:''' normalize each dimension in config to [0,1]'''- config_norm = {}- for key, value in flatten_dict(config).items():- if key in self.space:- # domain: sample.Categorical/Integer/Float/Function- domain = self.space[key]- if not callable(getattr(domain, 'get_sampler', None)):- config_norm[key] = value- else:- if isinstance(domain, sample.Categorical):- # normalize categorical- if key in self._ordered_cat_hp:- l, d = self._ordered_cat_hp[key]- config_norm[key] = (d[value] + 0.5) / len(l)- elif key in self._ordered_choice_hp:- l, d = self._ordered_choice_hp[key]- config_norm[key] = (d[value] + 0.5) / len(l)- elif key in self.incumbent:- config_norm[key] = self.incumbent[- key] if value == self.best_config[- key] else (- self.incumbent[key]- + 1.0 / self._unordered_cat_hp[key]) % 1- else:- config_norm[key] = 0.5- continue- # Uniform/LogUniform/Normal/Base- sampler = domain.get_sampler()- if isinstance(sampler, sample.Quantized):- # sampler is sample.Quantized- sampler = sampler.get_sampler()- if str(sampler) == 'LogUniform':- config_norm[key] = np.log(value / domain.lower) / np.log(- domain.upper / domain.lower)- elif str(sampler) == 'Uniform':- config_norm[key] = (- value - domain.lower) / (domain.upper - domain.lower)- elif str(sampler) == 'Normal':- # N(mean, sd) -> N(0,1)- config_norm[key] = (value - sampler.mean) / sampler.sd- else:- # TODO? elif str(sampler) == 'Base': # sample.Function._CallSampler- # e.g., {test: sample_from(lambda spec: randn(10, 2).sample() * 0.01)}- config_norm[key] = value- else: # prune_attr- config_norm[key] = value- return config_norm+ return normalize(+ config, self._space, self.best_config, self.incumbent, recursive)def denormalize(self, config):''' denormalize each dimension in config from [0,1]'''- config_denorm = {}- for key, value in config.items():- if key in self.space:- # domain: sample.Categorical/Integer/Float/Function- domain = self.space[key]- if not callable(getattr(domain, 'get_sampler', None)):- config_denorm[key] = value- else:- if isinstance(domain, sample.Categorical):- # denormalize categorical- if key in self._ordered_cat_hp:- l, _ = self._ordered_cat_hp[key]- n = len(l)- config_denorm[key] = l[min(n - 1, int(np.floor(value * n)))]- elif key in self._ordered_choice_hp:- l, _ = self._ordered_choice_hp[key]- n = len(l)- config_denorm[key] = l[min(n - 1, int(np.floor(value * n)))]- else:- assert key in self.incumbent- n = self._unordered_cat_hp[key]- if np.floor(value * n) == np.floor(self.incumbent[key] * n):- config_denorm[key] = self.best_config[key]- else: # ****random value each time!****- config_denorm[key] = self._random.choice(- [x for x in domain.categories- if x != self.best_config[key]])- continue- # Uniform/LogUniform/Normal/Base- sampler = domain.get_sampler()- if isinstance(sampler, sample.Quantized):- # sampler is sample.Quantized- sampler = sampler.get_sampler()- # Handle Log/Uniform- if str(sampler) == 'LogUniform':- config_denorm[key] = (- domain.upper / domain.lower) ** value * domain.lower- elif str(sampler) == 'Uniform':- config_denorm[key] = value * (- domain.upper - domain.lower) + domain.lower- elif str(sampler) == 'Normal':- # denormalization for 'Normal'- config_denorm[key] = value * sampler.sd + sampler.mean- else:- config_denorm[key] = value- # Handle quantized- sampler = domain.get_sampler()- if isinstance(sampler, sample.Quantized):- config_denorm[key] = np.round(- np.divide(config_denorm[key], sampler.q)) * sampler.q- # Handle int (4.6 -> 5)- if isinstance(domain, sample.Integer):- config_denorm[key] = int(round(config_denorm[key]))- else: # prune_attr- config_denorm[key] = value- return config_denorm+ return denormalize(+ config, self._space, self.best_config, self.incumbent, self._random)def set_search_properties(self,metric: Optional[str] = None,@@ -428,6 +285,7 @@ class FLOW2(Searcher):self.metric_op = 1.if config:self.space = config+ self._space = flatten_dict(self.space)self._init_search()return True@@ -600,7 +458,7 @@ class FLOW2(Searcher):for i, key in enumerate(self._tunable_keys):if self._direction_tried[i] != 0:for _, generated in generate_variants({'config': {- key: self.space[key]+ key: self._space[key]}}):if generated['config'][key] != best_config[key]:config[key] = generated['config'][key]@@ -632,26 +490,27 @@ class FLOW2(Searcher):'''return self._num_allowed4incumbent > 0- def config_signature(self, config) -> tuple:+ def config_signature(self, config, space: Dict = None) -> tuple:''' return the signature tuple of a config'''config = flatten_dict(config)+ if space:+ space = flatten_dict(space)+ else:+ space = self._spacevalue_list = []- for key in self._space_keys:- if key in config:- value = config[key]- if key == self.prune_attr:- value_list.append(value)- # else key must be in self.space- # get rid of list type or constant,- # e.g., "eval_metric": ["logloss", "error"]- elif callable(getattr(self.space[key], 'sample', None)):- if isinstance(self.space[key], sample.Integer):- value_list.append(int(round(value)))- else:- value_list.append(value)+ keys = sorted(config.keys()) if self._hierarchical else self._space_keys+ for key in keys:+ value = config[key]+ if key == self.prune_attr:+ value_list.append(value)+ # else key must be in self.space+ # get rid of list type or constant,+ # e.g., "eval_metric": ["logloss", "error"]+ elif isinstance(space[key], sample.Integer):+ value_list.append(int(round(value)))else:- value_list.append(None)+ value_list.append(value)return tuple(value_list)@propertydiff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.pyindex 8640e6132..9c6a7fb05 100644--- a/flaml/searcher/search_thread.py+++ b/flaml/searcher/search_thread.py@@ -6,10 +6,13 @@from typing import Dict, Optionalimport numpy as nptry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.suggest import Searcher-except ImportError:+except (ImportError, AssertionError):from .suggestion import Searcherfrom .flow2 import FLOW2+from ..tune.space import (add_cost_to_space, unflatten_hierarchical)import logginglogger = logging.getLogger(__name__)@@ -41,6 +44,12 @@ class SearchThread:self._init_config = Trueself.running = 0 # the number of running trials from the threadself.cost_attr = cost_attr+ if search_alg:+ self.space = self._space = search_alg.space # unflattened space+ # TODO: remove when define_by_run is supported+ if not isinstance(self._search_alg, FLOW2):+ # remember const config+ self._const = add_cost_to_space(self.space, {}, {})@classmethoddef set_eps(cls, time_budget_s):@@ -54,6 +63,9 @@ class SearchThread:else:try:config = self._search_alg.suggest(trial_id)+ # TODO: remove when define_by_run is supported+ config.update(self._const)+ config, self.space = unflatten_hierarchical(config, self._space)except FloatingPointError:logger.warning('The global search method raises FloatingPointError. 'diff --git a/flaml/tune/__init__.py b/flaml/tune/__init__.pyindex 6a581608b..111b1a532 100644--- a/flaml/tune/__init__.py+++ b/flaml/tune/__init__.py@@ -1,9 +1,11 @@try:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform, lograndint)-except ImportError:+ qrandn, loguniform, qloguniform, lograndint, qlograndint)+except (ImportError, AssertionError):from .sample import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform, lograndint)+ qrandn, loguniform, qloguniform, lograndint, qlograndint)from .tune import run, reportfrom .sample import polynomial_expansion_setfrom .sample import PolynomialExpansionSet, Categorical, Floatdiff --git a/flaml/tune/cgmanifest.json b/flaml/tune/cgmanifest.jsonindex 96626929b..03a3377cc 100644--- a/flaml/tune/cgmanifest.json+++ b/flaml/tune/cgmanifest.json@@ -6,6 +6,6 @@"pip": {"Name": "ray[tune]", "Version": "1.5.1" }},"DevelopmentDependency": false- },+ }]}\ No newline at end of filediff --git a/flaml/tune/space.py b/flaml/tune/space.pyindex 0ffbedac1..ff4d5fa98 100644--- a/flaml/tune/space.py+++ b/flaml/tune/space.py@@ -1,9 +1,14 @@try:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune import sample-except ImportError:+ from ray.tune.suggest.variant_generator import generate_variants+except (ImportError, AssertionError):from . import sample-from typing import Dict, Optional, Any+ from ..searcher.variant_generator import generate_variants+from typing import Dict, Optional, Any, Tuple+import numpy as npimport logginglogger = logging.getLogger(__name__)@@ -15,7 +20,7 @@ def define_by_run_func("""Define-by-run function to create the search space.Returns:- None or a dict with constant values.+ A dict with constant values."""config = {}for key, domain in space.items():@@ -50,12 +55,16 @@ def define_by_run_func(elif isinstance(domain, sample.Integer):if isinstance(sampler, sample.LogUniform):trial.suggest_int(- key, domain.lower, domain.upper, step=quantize or 1, log=True)+ key, domain.lower,+ domain.upper - int(bool(not quantize)),+ step=quantize or 1, log=True)elif isinstance(sampler, sample.Uniform):# Upper bound should be inclusive for quantization and# exclusive otherwisetrial.suggest_int(- key, domain.lower, domain.upper, step=quantize or 1)+ key, domain.lower,+ domain.upper - int(bool(not quantize)),+ step=quantize or 1)elif isinstance(domain, sample.Categorical):if isinstance(sampler, sample.Uniform):if not hasattr(domain, 'choices'):@@ -76,3 +85,352 @@ def define_by_run_func(type(domain.sampler).__name__))# Return all constants in a dictionary.return config+++def unflatten_hierarchical(config: Dict, space: Dict) -> Tuple[Dict, Dict]:+ '''unflatten hierarchical config'''+ hier = {}+ subspace = {}+ for key, value in config.items():+ if '/' in key:+ key = key[key.rfind('/') + 1:]+ if ':' in key:+ pos = key.rfind(':')+ true_key = key[:pos]+ choice = int(key[pos + 1:])+ hier[true_key], subspace[true_key] = unflatten_hierarchical(+ value, space[true_key][choice])+ else:+ domain = space.get(key)+ if domain is not None:+ subspace[key] = domain+ if isinstance(domain, sample.Domain):+ sampler = domain.sampler+ if isinstance(sampler, sample.Quantized):+ q = sampler.q+ sampler = sampler.sampler+ if isinstance(sampler, sample.LogUniform):+ value = domain.cast(np.round(value / q) * q)+ hier[key] = value+ return hier, subspace+++def add_cost_to_space(space: Dict, low_cost_point: Dict, choice_cost: Dict):+ """Update the space in place by adding low_cost_point and choice_cost++ Returns:+ A dict with constant values.+ """+ config = {}+ for key in space:+ domain = space[key]+ if not isinstance(domain, sample.Domain):+ if isinstance(domain, dict):+ low_cost = low_cost_point.get(key, {})+ choice_cost_list = choice_cost.get(key, {})+ const = add_cost_to_space(+ domain, low_cost, choice_cost_list)+ if const:+ config[key] = const+ else:+ config[key] = domain+ continue+ low_cost = low_cost_point.get(key)+ choice_cost_list = choice_cost.get(key)+ if callable(getattr(domain, 'get_sampler', None)):+ sampler = domain.get_sampler()+ if isinstance(sampler, sample.Quantized):+ sampler = sampler.get_sampler()+ domain.bounded = str(sampler) != 'Normal'+ if isinstance(domain, sample.Categorical):+ domain.const = []+ for i, cat in enumerate(domain.categories):+ if isinstance(cat, dict):+ if isinstance(low_cost, list):+ low_cost_dict = low_cost[i]+ else:+ low_cost_dict = {}+ if choice_cost_list:+ choice_cost_dict = choice_cost_list[i]+ else:+ choice_cost_dict = {}+ domain.const.append(add_cost_to_space(+ cat, low_cost_dict, choice_cost_dict))+ else:+ domain.const.append(None)+ if choice_cost_list:+ if len(choice_cost_list) == len(domain.categories):+ domain.choice_cost = choice_cost_list+ else:+ domain.choice_cost = choice_cost_list[-1]+ # sort the choices by cost+ cost = np.array(domain.choice_cost)+ ind = np.argsort(cost)+ domain.categories = [domain.categories[i] for i in ind]+ domain.choice_cost = cost[ind]+ domain.const = [domain.const[i] for i in ind]+ domain.ordered = True+ elif all(isinstance(x, int) or isinstance(x, float)+ for x in domain.categories):+ # sort the choices by value+ ind = np.argsort(domain.categories)+ domain.categories = [domain.categories[i] for i in ind]+ domain.ordered = True+ else:+ domain.ordered = False+ if low_cost and low_cost not in domain.categories:+ assert isinstance(low_cost, list), \+ f"low cost {low_cost} not in domain {domain.categories}"+ if domain.ordered:+ sorted_points = [low_cost[i] for i in ind]+ for i, point in enumerate(sorted_points):+ low_cost[i] = point+ if len(low_cost) > len(domain.categories):+ if domain.ordered:+ low_cost[-1] = int(np.where(ind == low_cost[-1])[0])+ domain.low_cost_point = low_cost[-1]+ return+ if low_cost:+ domain.low_cost_point = low_cost+ return config+++def normalize(+ config: Dict, space: Dict, reference_config: Dict,+ normalized_reference_config: Dict, recursive: bool = False,+):+ '''normalize config in space according to reference_config.+ normalize each dimension in config to [0,1].+ '''+ config_norm = {}+ for key in config:+ value = config[key]+ domain = space.get(key)+ if domain is None: # e.g., prune_attr+ config_norm[key] = value+ continue+ if not callable(getattr(domain, 'get_sampler', None)):+ if recursive and isinstance(domain, dict):+ config_norm[key] = normalize(+ value, domain, reference_config[key], {})+ else:+ config_norm[key] = value+ continue+ # domain: sample.Categorical/Integer/Float/Function+ if isinstance(domain, sample.Categorical):+ norm = None+ # value is either one category, or the low_cost_point list+ if value not in domain.categories:+ # nested, low_cost_point list+ if recursive:+ norm = []+ for i, cat in enumerate(domain.categories):+ norm.append(normalize(+ value[i], cat, reference_config[key][i], {}))+ if isinstance(value, list) and len(value) > len(+ domain.categories):+ # low_cost_point list+ index = value[-1]+ config[key] = value[index]+ value = domain.categories[index]+ else:+ continue+ # normalize categorical+ n = len(domain.categories)+ if domain.ordered:+ normalized = (domain.categories.index(value) + 0.5) / n+ elif key in normalized_reference_config:+ normalized = normalized_reference_config[+ key] if value == reference_config[key] else (+ normalized_reference_config[key] + 1 / n) % 1+ else:+ normalized = 0.5+ if norm:+ norm.append(normalized)+ else:+ norm = normalized+ config_norm[key] = norm+ continue+ # Uniform/LogUniform/Normal/Base+ sampler = domain.get_sampler()+ if isinstance(sampler, sample.Quantized):+ # sampler is sample.Quantized+ quantize = sampler.q+ sampler = sampler.get_sampler()+ else:+ quantize = None+ if str(sampler) == 'LogUniform':+ upper = domain.upper - (+ isinstance(domain, sample.Integer) & (quantize is None))+ config_norm[key] = np.log(value / domain.lower) / np.log(+ upper / domain.lower)+ elif str(sampler) == 'Uniform':+ upper = domain.upper - (+ isinstance(domain, sample.Integer) & (quantize is None))+ config_norm[key] = (value - domain.lower) / (upper - domain.lower)+ elif str(sampler) == 'Normal':+ # N(mean, sd) -> N(0,1)+ config_norm[key] = (value - sampler.mean) / sampler.sd+ else:+ # TODO? elif str(sampler) == 'Base': # sample.Function._CallSampler+ # e.g., {test: sample_from(lambda spec: randn(10, 2).sample() * 0.01)}+ config_norm[key] = value+ return config_norm+++def denormalize(+ config: Dict, space: Dict, reference_config: Dict,+ normalized_reference_config: Dict, random_state+):+ config_denorm = {}+ for key, value in config.items():+ if key in space:+ # domain: sample.Categorical/Integer/Float/Function+ domain = space[key]+ if not callable(getattr(domain, 'get_sampler', None)):+ config_denorm[key] = value+ else:+ if isinstance(domain, sample.Categorical):+ # denormalize categorical+ n = len(domain.categories)+ if domain.ordered:+ config_denorm[key] = domain.categories[+ min(n - 1, int(np.floor(value * n)))]+ else:+ assert key in normalized_reference_config+ if np.floor(value * n) == np.floor(+ normalized_reference_config[key] * n):+ config_denorm[key] = reference_config[key]+ else: # ****random value each time!****+ config_denorm[key] = random_state.choice(+ [x for x in domain.categories+ if x != reference_config[key]])+ continue+ # Uniform/LogUniform/Normal/Base+ sampler = domain.get_sampler()+ if isinstance(sampler, sample.Quantized):+ # sampler is sample.Quantized+ quantize = sampler.q+ sampler = sampler.get_sampler()+ else:+ quantize = None+ # Handle Log/Uniform+ if str(sampler) == 'LogUniform':+ upper = domain.upper - (isinstance(domain, sample.Integer)+ & (quantize is None))+ config_denorm[key] = (+ upper / domain.lower) ** value * domain.lower+ elif str(sampler) == 'Uniform':+ upper = domain.upper - (isinstance(domain, sample.Integer)+ & (quantize is None))+ config_denorm[key] = value * (+ upper - domain.lower) + domain.lower+ elif str(sampler) == 'Normal':+ # denormalization for 'Normal'+ config_denorm[key] = value * sampler.sd + sampler.mean+ else:+ config_denorm[key] = value+ # Handle quantized+ if quantize is not None:+ config_denorm[key] = np.round(+ np.divide(config_denorm[key], quantize)) * quantize+ # Handle int (4.6 -> 5)+ if isinstance(domain, sample.Integer):+ config_denorm[key] = int(round(config_denorm[key]))+ else: # prune_attr+ config_denorm[key] = value+ return config_denorm+++def indexof(domain: Dict, config: Dict) -> int:+ '''find the index of config in domain.categories+ '''+ index = config.get('_choice_')+ if index is not None:+ return index+ if config in domain.categories:+ return domain.categories.index(config)+ # print(config)+ for i, cat in enumerate(domain.categories):+ # print(cat)+ if not isinstance(cat, dict):+ continue+ # print(len(cat), len(config))+ if len(cat) != len(config):+ continue+ # print(cat.keys())+ if not set(cat.keys()).issubset(set(config.keys())):+ continue+ # print(domain.const[i])+ if all(config[key] == value for key, value in domain.const[i].items()):+ return i+ return None+++def complete_config(+ partial_config: Dict, space: Dict, flow2, disturb: bool = False,+ lower: Optional[Dict] = None, upper: Optional[Dict] = None+) -> Tuple[Dict, Dict]:+ '''Complete partial config in space++ Returns:+ config, space+ '''+ config = partial_config.copy()+ normalized = normalize(config, space, config, {})+ if disturb:+ for key in normalized:+ domain = space.get(key)+ if getattr(domain, 'ordered', True) is False:+ # don't change unordered cat choice+ continue+ if not callable(getattr(domain, 'get_sampler', None)):+ continue+ if upper and lower:+ up, low = upper[key], lower[key]+ gauss_std = up - low or flow2.STEPSIZE+ # allowed bound+ up += flow2.STEPSIZE+ low -= flow2.STEPSIZE+ elif domain.bounded:+ up, low, gauss_std = 1, 0, 1.0+ else:+ up, low, gauss_std = np.Inf, -np.Inf, 1.0+ if domain.bounded:+ up = min(up, 1)+ low = max(low, 0)+ delta = flow2.rand_vector_gaussian(1, gauss_std)[0]+ normalized[key] = max(low, min(up, normalized[key] + delta))+ config = denormalize(normalized, space, config, normalized, flow2._random)+ for key, value in space.items():+ if key not in config:+ config[key] = value+ for _, generated in generate_variants({'config': config}):+ config = generated['config']+ break+ subspace = {}+ for key, domain in space.items():+ value = config[key]+ if isinstance(value, dict):+ if isinstance(domain, sample.Categorical):+ # nested space+ index = indexof(domain, value)+ # point = partial_config.get(key)+ # if isinstance(point, list): # low cost point list+ # point = point[index]+ # else:+ # point = {}+ config[key], subspace[key] = complete_config(+ value, domain.categories[index], flow2, disturb,+ lower and lower[key][index], upper and upper[key][index]+ )+ assert '_choice_' not in subspace[key], \+ "_choice_ is a reserved key for hierarchical search space"+ subspace[key]['_choice_'] = index+ else:+ config[key], subspace[key] = complete_config(+ value, space[key], flow2, disturb,+ lower and lower[key], upper and upper[key])+ continue+ subspace[key] = domain+ return config, subspacediff --git a/flaml/tune/trial.py b/flaml/tune/trial.pyindex ad15edb02..394c1ffaa 100644--- a/flaml/tune/trial.py+++ b/flaml/tune/trial.py@@ -21,11 +21,10 @@ import uuidimport timefrom numbers import Numberfrom collections import deque-import copydef flatten_dict(dt, delimiter="/", prevent_delimiter=False):- dt = copy.deepcopy(dt)+ dt = dt.copy()if prevent_delimiter and any(delimiter in key for key in dt):# Raise if delimiter is any of the keysraise ValueError(diff --git a/flaml/tune/trial_runner.py b/flaml/tune/trial_runner.pyindex 9f0e8c191..690e57df3 100644--- a/flaml/tune/trial_runner.py+++ b/flaml/tune/trial_runner.py@@ -4,10 +4,12 @@* project root for license information.'''from typing import Optional-try:- from ray.tune.trial import Trial-except ImportError:- from .trial import Trial+# try:+# from ray import __version__ as ray_version+# assert ray_version >= '1.0.0'+# from ray.tune.trial import Trial+# except (ImportError, AssertionError):+from .trial import Trialimport logginglogger = logging.getLogger(__name__)diff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex fe4ac1cd5..a46054869 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -8,8 +8,10 @@ import numpy as npimport datetimeimport timetry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.analysis import ExperimentAnalysis as EA-except ImportError:+except (ImportError, AssertionError):from .analysis import ExperimentAnalysis as EAimport logginglogger = logging.getLogger(__name__)@@ -288,9 +290,11 @@ def run(training_function,if reduction_factor:params['reduction_factor'] = reduction_factortry:+ from ray import __version__ as ray_version+ assert ray_version >= '1.0.0'from ray.tune.schedulers import ASHASchedulerscheduler = ASHAScheduler(**params)- except ImportError:+ except (ImportError, AssertionError):passif use_ray:try:diff --git a/flaml/version.py b/flaml/version.pyindex 57f9f92e6..e1e093c04 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.5.11"+__version__ = "0.5.12"diff --git a/test/hf/test_deberta.py b/test/hf/test_deberta.pyindex 92afe51f8..18c392785 100644--- a/test/hf/test_deberta.py+++ b/test/hf/test_deberta.py@@ -52,6 +52,11 @@ try:except ImportError:print("pip install torch transformers datasets flaml[blendsearch,ray]")+except ValueError as e:+ if 'Connection error' in str(e):+ print(e)+ else:+ raise(e)import logginglogger = logging.getLogger(__name__)diff --git a/test/hf/test_distillbert.py b/test/hf/test_distillbert.pyindex 1f1945517..a81e5fece 100644--- a/test/hf/test_distillbert.py+++ b/test/hf/test_distillbert.py@@ -35,6 +35,11 @@ try:except ImportError:print("pip install torch transformers datasets flaml[blendsearch,ray]")+except ValueError as e:+ if 'Connection error' in str(e):+ print(e)+ else:+ raise(e)logger = logging.getLogger(__name__)os.makedirs('logs', exist_ok=True)diff --git a/test/hf/test_electra.py b/test/hf/test_electra.pyindex 2255452b4..b85e18c09 100644--- a/test/hf/test_electra.py+++ b/test/hf/test_electra.py@@ -52,6 +52,11 @@ try:except ImportError:print("pip install torch transformers datasets flaml[blendsearch,ray]")+except ValueError as e:+ if 'Connection error' in str(e):+ print(e)+ else:+ raise(e)import logginglogger = logging.getLogger(__name__)diff --git a/test/test_automl.py b/test/test_automl.pyindex e7ef9c6b3..7d5b2df16 100644--- a/test/test_automl.py+++ b/test/test_automl.py@@ -156,11 +156,42 @@ class TestAutoML(unittest.TestCase):X = pd.DataFrame({'f1': [1, -2, 3, -4, 5, -6, -7, 8, -9, -10, -11, -12, -13, -14],'f2': [3., 16., 10., 12., 3., 14., 11., 12., 5., 14., 20., 16., 15., 11.],- 'f3': ['a', 'b', 'a', 'c', 'c', 'b', 'b', 'b', 'b', 'a', 'b', 'e', 'e', 'a'],+ 'f3': ['a', 'b', 'a', 'c', 'c', 'b', 'b', 'b', 'b', 'a', 'b', 1.0, 1.0, 'a'],'f4': [True, True, False, True, True, False, False, False, True, True, False, False, True, True],})y = pd.Series([0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1])+ automl = AutoML()+ automl_settings = {+ "time_budget": 6,+ "task": 'classification',+ "n_jobs": 1,+ "estimator_list": ['catboost', 'lrl2'],+ "eval_method": "cv",+ "n_splits": 3,+ "metric": "accuracy",+ "log_training_metric": True,+ "verbose": 1,+ "ensemble": True,+ }+ automl.fit(X, y, **automl_settings)++ automl = AutoML()+ automl_settings = {+ "time_budget": 2,+ "task": 'classification',+ "n_jobs": 1,+ "estimator_list": ['lrl2', 'kneighbor'],+ "eval_method": "cv",+ "n_splits": 3,+ "metric": "accuracy",+ "log_training_metric": True,+ "verbose": 1,+ "ensemble": True,+ }+ automl.fit(X, y, **automl_settings)++ automl = AutoML()automl_settings = {"time_budget": 3,"task": 'classification',@@ -175,6 +206,21 @@ class TestAutoML(unittest.TestCase):}automl.fit(X, y, **automl_settings)+ automl = AutoML()+ automl_settings = {+ "time_budget": 3,+ "task": 'classification',+ "n_jobs": 1,+ "estimator_list": ['lgbm', 'catboost', 'kneighbor'],+ "eval_method": "cv",+ "n_splits": 3,+ "metric": "accuracy",+ "log_training_metric": True,+ "verbose": 1,+ "ensemble": True,+ }+ automl.fit(X, y, **automl_settings)+def test_dataframe(self):self.test_classification(True)diff --git a/test/test_notebook_example.py b/test/test_notebook_example.pyindex a669a65cb..12056ba26 100644--- a/test/test_notebook_example.py+++ b/test/test_notebook_example.py@@ -1,6 +1,14 @@-def test_automl(budget=5):+from openml.exceptions import OpenMLServerException+++def test_automl(budget=5, dataset_format='dataframe'):from flaml.data import load_openml_dataset- X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir='test/')+ try:+ X_train, X_test, y_train, y_test = load_openml_dataset(+ dataset_id=1169, data_dir='test/', dataset_format=dataset_format)+ except OpenMLServerException:+ print("OpenMLServerException raised")+ return''' import AutoML class from flaml package '''from flaml import AutoMLautoml = AutoML()@@ -42,13 +50,22 @@ def test_automl(budget=5):print(automl.min_resource)+def test_automl_array():+ test_automl(5, 'array')++def test_mlflow():import subprocessimport syssubprocess.check_call([sys.executable, "-m", "pip", "install", "mlflow"])import mlflowfrom flaml.data import load_openml_task- X_train, X_test, y_train, y_test = load_openml_task(task_id=7592, data_dir='test/')+ try:+ X_train, X_test, y_train, y_test = load_openml_task(+ task_id=7592, data_dir='test/')+ except OpenMLServerException:+ print("OpenMLServerException raised")+ return''' import AutoML class from flaml package '''from flaml import AutoMLautoml = AutoML()diff --git a/test/test_python_log.py b/test/test_python_log.pyindex 732c881d1..d5027d596 100644--- a/test/test_python_log.py+++ b/test/test_python_log.py@@ -1,3 +1,4 @@+from flaml.tune.space import unflatten_hierarchicalfrom flaml import AutoMLfrom sklearn.datasets import load_bostonimport os@@ -45,13 +46,38 @@ class TestLogging(unittest.TestCase):logger.info(automl.search_space)logger.info(automl.low_cost_partial_config)logger.info(automl.points_to_evalaute)+ logger.info(automl.cat_hp_cost)import optuna as otstudy = ot.create_study()- from flaml.tune.space import define_by_run_func- logger.info(define_by_run_func(study.ask(), automl.search_space))+ from flaml.tune.space import define_by_run_func, add_cost_to_space+ sample = define_by_run_func(study.ask(), automl.search_space)+ logger.info(sample)+ logger.info(unflatten_hierarchical(sample, automl.search_space))+ add_cost_to_space(+ automl.search_space, automl.low_cost_partial_config,+ automl.cat_hp_cost+ )+ logger.info(automl.search_space["ml"].categories)config = automl.best_config.copy()config['learner'] = automl.best_estimatorautoml.trainable({"ml": config})+ from flaml import tune, CFO+ search_alg = CFO(+ metric='val_loss',+ space=automl.search_space,+ low_cost_partial_config=automl.low_cost_partial_config,+ points_to_evaluate=automl.points_to_evalaute,+ cat_hp_cost=automl.cat_hp_cost,+ prune_attr=automl.prune_attr,+ min_resource=automl.min_resource,+ max_resource=automl.max_resource,+ config_constraints=[(automl.size, '<=', automl._mem_thres)],+ metric_constraints=automl.metric_constraints)+ analysis = tune.run(+ automl.trainable, search_alg=search_alg, # verbose=2,+ time_budget_s=1, num_samples=-1)+ print(min((trial.last_result["val_loss"], trial.last_result)+ for trial in analysis.trials))# Check if the log buffer is populated.self.assertTrue(len(buf.getvalue()) > 0)diff --git a/test/test_training_log.py b/test/test_training_log.pyindex 8895d5cfd..1b5952430 100644--- a/test/test_training_log.py+++ b/test/test_training_log.py@@ -10,10 +10,10 @@ from flaml.training_log import training_log_readerclass TestTrainingLog(unittest.TestCase):- def test_training_log(self):+ def test_training_log(self, path='test_training_log.log'):with TemporaryDirectory() as d:- filename = os.path.join(d, 'test_training_log.log')+ filename = os.path.join(d, path)# Run a simple job.automl_experiment = AutoML()@@ -42,3 +42,15 @@ class TestTrainingLog(unittest.TestCase):print(record)count += 1self.assertGreater(count, 0)++ automl_settings["log_file_name"] = None+ automl_experiment.fit(X_train=X_train, y_train=y_train,+ **automl_settings)++ def test_illfilename(self):+ try:+ self.test_training_log('/')+ except IsADirectoryError:+ print("IsADirectoryError happens as expected in linux.")+ except PermissionError:+ print("PermissionError happens as expected in windows.")diff --git a/test/test_xgboost2d.py b/test/test_xgboost2d.pyindex c9f380c67..a98ec00ba 100644--- a/test/test_xgboost2d.py+++ b/test/test_xgboost2d.py@@ -59,7 +59,7 @@ def test_simple(method=None):automl.trainable(config)from flaml import tuneanalysis = tune.run(- automl.trainable, automl.search_space, metric='val_loss',+ automl.trainable, automl.search_space, metric='val_loss', mode="min",low_cost_partial_config=automl.low_cost_partial_config,points_to_evaluate=automl.points_to_evalaute,cat_hp_cost=automl.cat_hp_cost,@@ -68,7 +68,7 @@ def test_simple(method=None):max_resource=automl.max_resource,time_budget_s=automl._state.time_budget,config_constraints=[(automl.size, '<=', automl._mem_thres)],- metric_constraints=automl.metric_constraints)+ metric_constraints=automl.metric_constraints, num_samples=5)print(analysis.trials[-1])diff --git a/test/tune/test_tune.py b/test/tune/test_tune.pyindex 1d0f96032..ded2b294f 100644--- a/test/tune/test_tune.py+++ b/test/tune/test_tune.py@@ -52,7 +52,7 @@ def _test_xgboost(method='BlendSearch'):else:from ray import tunesearch_space = {- "max_depth": tune.randint(1, 8) if method in [+ "max_depth": tune.randint(1, 9) if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),"min_child_weight": tune.choice([1, 2, 3]),"subsample": tune.uniform(0.5, 1.0),@@ -61,7 +61,7 @@ def _test_xgboost(method='BlendSearch'):max_iter = 10for num_samples in [128]:time_budget_s = 60- for n_cpu in [8]:+ for n_cpu in [4]:start_time = time.time()ray.init(num_cpus=n_cpu, num_gpus=0)# ray.init(address='auto')@@ -168,7 +168,7 @@ def test_nested():search_space = {# test nested search space"cost_related": {- "a": tune.randint(1, 8),+ "a": tune.randint(1, 9),},"b": tune.uniform(0.5, 1.0),}@@ -194,7 +194,7 @@ def test_nested():metric_constraints=[("ab", "<=", 4)]),local_dir='logs/',num_samples=-1,- time_budget_s=.1)+ time_budget_s=1)best_trial = analysis.get_best_trial()logger.info(f"CFO best config: {best_trial.config}")@@ -216,7 +216,7 @@ def test_nested():metric_constraints=[("ab", "<=", 4)]),local_dir='logs/',num_samples=-1,- time_budget_s=.1)+ time_budget_s=1)best_trial = analysis.get_best_trial()logger.info(f"BlendSearch exp best config: {best_trial.config}")@@ -233,7 +233,7 @@ def test_nested():metric_constraints=[("ab", "<=", 4)],local_dir='logs/',num_samples=-1,- time_budget_s=.1)+ time_budget_s=1)best_trial = analysis.get_best_trial()logger.info(f"BlendSearch best config: {best_trial.config}")@@ -251,10 +251,11 @@ def test_run_training_function_return_value():tune.run(evaluate_config_dict,config={- 'x': tune.lograndint(lower=1, upper=100000),- 'y': tune.randint(lower=1, upper=100000)+ 'x': tune.qloguniform(lower=1, upper=100000, q=1),+ 'y': tune.qrandint(lower=2, upper=100000, q=2)},- metric='metric',+ metric='metric', mode='max',+ num_samples=100,)# Test scalar return value@@ -265,9 +266,10 @@ def test_run_training_function_return_value():tune.run(evaluate_config_scalar,config={- 'x': tune.lograndint(lower=1, upper=100000),- 'y': tune.randint(lower=1, upper=100000)+ 'x': tune.qloguniform(lower=1, upper=100000, q=1),+ 'y': tune.qlograndint(lower=2, upper=100000, q=2)},+ num_samples=100, mode='max',)