Skip to content

A programming framework for agentic AI

space -> main (#148)

Xueqing Liueeaf5b596308
diff --git a/flaml/automl.py b/flaml/automl.py
index f32943956..1e4fb3aa4 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -4,6 +4,7 @@
* project root for license information.
'''
import time
+from typing import Callable, Optional
import warnings
from functools import partial
import numpy as np
@@ -212,10 +213,11 @@ class AutoMLState:
'val_loss': val_loss,
'trained_estimator': trained_estimator
}
- with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
- tune.report(**result)
if sampled_weight is not None:
self.fit_kwargs['sample_weight'] = weight
+ # with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+ # tune.report(**result)
+ return result
def _train_with_config(
self, estimator, config_w_resource, sample_size=None
@@ -790,6 +792,177 @@ class AutoML:
else:
return 'holdout'
+ @property
+ def search_space(self) -> dict:
+ '''Search space
+ Must be called after fit(...) (use max_iter=0 to prevent actual fitting)
+
+ Returns:
+ A dict of the search space
+ '''
+ estimator_list = self.estimator_list
+ if len(estimator_list) == 1:
+ estimator = estimator_list[0]
+ space = self._search_states[estimator].search_space.copy()
+ space['learner'] = estimator
+ return space
+ choices = []
+ for estimator in estimator_list:
+ space = self._search_states[estimator].search_space.copy()
+ space['learner'] = estimator
+ choices.append(space)
+ return {'ml': tune.choice(choices)}
+
+ @property
+ def low_cost_partial_config(self) -> dict:
+ '''Low cost partial config
+
+ Returns:
+ A dict.
+ (a) if there is only one estimator in estimator_list, each key is a
+ hyperparameter name
+ (b) otherwise, it is a nested dict with 'ml' as the key, and
+ a list of the low_cost_partial_configs as the value, corresponding
+ to each learner's low_cost_partial_config
+
+ '''
+ if len(self.estimator_list) == 1:
+ estimator = self.estimator_list[0]
+ c = self._search_states[estimator].low_cost_partial_config
+ return c
+ else:
+ configs = []
+ for estimator in self.estimator_list:
+ c = self._search_states[estimator].low_cost_partial_config
+ configs.append(c)
+ config = {'ml': configs}
+ return config
+
+ @property
+ def cat_hp_cost(self) -> dict:
+ '''Categorical hyperparameter cost
+
+ Returns:
+ A dict.
+ (a) if there is only one estimator in estimator_list, each key is a
+ hyperparameter name
+ (b) otherwise, it is a nested dict with 'ml' as the key, and
+ a list of the cat_hp_cost's as the value, corresponding
+ to each learner's cat_hp_cost
+
+ '''
+ if len(self.estimator_list) == 1:
+ estimator = self.estimator_list[0]
+ c = self._search_states[estimator].cat_hp_cost
+ return c
+ else:
+ configs = []
+ for estimator in self.estimator_list:
+ c = self._search_states[estimator].cat_hp_cost
+ configs.append(c)
+ config = {'ml': configs}
+ return config
+
+ @property
+ def points_to_evalaute(self) -> dict:
+ '''Initial points to evaluate
+
+ Returns:
+ A list of dicts. Each dict is the initial point for each learner
+ '''
+ points = []
+ for estimator in self.estimator_list:
+ config = self._search_states[estimator].init_config
+ config['learner'] = estimator
+ if len(self.estimator_list) > 1:
+ points.append({'ml': config})
+ else:
+ points.append(config)
+ return points
+
+ @property
+ def prune_attr(self) -> Optional[str]:
+ '''Attribute for pruning
+
+ Returns:
+ A string for the sample size attribute or None
+ '''
+ return 'FLAML_sample_size' if self._sample else None
+
+ @property
+ def min_resource(self) -> Optional[float]:
+ '''Attribute for pruning
+
+ Returns:
+ A float for the minimal sample size or None
+ '''
+ return MIN_SAMPLE_TRAIN if self._sample else None
+
+ @property
+ def max_resource(self) -> Optional[float]:
+ '''Attribute for pruning
+
+ Returns:
+ A float for the maximal sample size or None
+ '''
+ return self._state.data_size if self._sample else None
+
+ @property
+ def trainable(self) -> Callable[[dict], Optional[float]]:
+ '''Training function
+
+ Returns:
+ A function that evaluates each config and returns the loss
+ '''
+ self._state.time_from_start = 0
+ for estimator in self.estimator_list:
+ search_state = self._search_states[estimator]
+ if not hasattr(search_state, 'training_function'):
+ search_state.training_function = partial(
+ AutoMLState._compute_with_config_base,
+ self._state, estimator)
+ states = self._search_states
+
+ def train(config: dict):
+ sample_size = config.get('FLAML_sample_size')
+ config = config.get('ml', config).copy()
+ if sample_size:
+ config['FLAML_sample_size'] = sample_size
+ estimator = config['learner']
+ del config['learner']
+ states[estimator].training_function(config)
+
+ return train
+
+ @property
+ def size(self) -> Callable[[dict], float]:
+ '''Size function
+
+ Returns:
+ A function that returns the mem size in bytes for a config
+ '''
+
+ def size_func(config: dict) -> float:
+ config = config.get('ml', config).copy
+ estimator = config['learner']
+ learner_class = self._state.learner_classes.get(estimator)
+ return learner_class.size(config)
+
+ return size_func
+
+ @property
+ def metric_constraints(self) -> list:
+ '''Metric constraints
+
+ Returns:
+ A list of the metric constraints
+ '''
+ constraints = []
+ if np.isfinite(self._pred_time_limit):
+ constraints.append(
+ ('pred_time', '<=', self._pred_time_limit))
+ return constraints
+
def fit(self,
X_train=None,
y_train=None,
@@ -969,11 +1142,12 @@ class AutoML:
)
logger.info("List of ML learners in AutoML Run: {}".format(
estimator_list))
+ self.estimator_list = estimator_list
self._hpo_method = hpo_method or 'cfo'
with training_log_writer(log_file_name) as save_helper:
self._training_log = save_helper
self._state.time_budget = time_budget
- self.estimator_list = estimator_list
+ self._active_estimators = estimator_list.copy()
self._ensemble = ensemble
self._max_iter = max_iter
self._mem_thres = mem_thres
@@ -1028,9 +1202,9 @@ class AutoML:
for self._track_iter in range(self._max_iter):
if self._estimator_index is None:
- estimator = self.estimator_list[0]
+ estimator = self._active_estimators[0]
else:
- estimator = self._select_estimator(self.estimator_list)
+ estimator = self._select_estimator(self._active_estimators)
if not estimator:
break
logger.info(
@@ -1071,10 +1245,6 @@ class AutoML:
points_to_evaluate = [search_state.init_config]
low_cost_partial_config = search_state.low_cost_partial_config
if self._hpo_method in ('bs', 'cfo', 'grid'):
- metric_constraints = []
- if np.isfinite(self._pred_time_limit):
- metric_constraints.append(
- ('pred_time', '<=', self._pred_time_limit))
algo = SearchAlgo(
metric='val_loss', mode='min', space=search_space,
points_to_evaluate=points_to_evaluate,
@@ -1086,7 +1256,7 @@ class AutoML:
config_constraints=[
(learner_class.size, '<=', self._mem_thres)
],
- metric_constraints=metric_constraints,
+ metric_constraints=self.metric_constraints,
)
else:
algo = SearchAlgo(
@@ -1198,7 +1368,7 @@ class AutoML:
else:
logger.info(f"no enough budget for learner {estimator}")
if self._estimator_index is not None:
- self.estimator_list.remove(estimator)
+ self._active_estimators.remove(estimator)
self._estimator_index -= 1
if self._retrain_full and best_config_sig and not better and (
self._search_states[
@@ -1217,7 +1387,7 @@ class AutoML:
est_retrain_time = 0
self._state.time_from_start = time.time() - self._start_time_flag
if (self._state.time_from_start >= self._state.time_budget
- or not self.estimator_list):
+ or not self._active_estimators):
break
if self._ensemble and self._best_estimator:
time_left = self._state.time_budget - self._state.time_from_start
diff --git a/flaml/nlp/README.md b/flaml/nlp/README.md
index c0da3cf7b..3ff6b33b9 100644
--- a/flaml/nlp/README.md
+++ b/flaml/nlp/README.md
@@ -1,6 +1,6 @@
# Hyperparameter Optimization for Huggingface Transformers
-AutoTransformers is an AutoML class for fine-tuning pre-trained language models based on the transformers library.
+AutoTransformers is an AutoML class for fine-tuning pre-trained language models based on the transformers library.
An example of using AutoTransformers:
@@ -9,36 +9,46 @@ from flaml.nlp.autotransformers import AutoTransformers
autohf = AutoTransformers()
preparedata_setting = {
- "dataset_subdataset_name": "glue:mrpc",
- "pretrained_model_size": "electra-base-discriminator:base",
- "data_root_path": "data/",
- "max_seq_length": 128,
- }
+ "dataset_subdataset_name": "glue:mrpc",
+ "pretrained_model_size": "electra-base-discriminator:base",
+ "data_root_path": "data/",
+ "max_seq_length": 128,
+}
autohf.prepare_data(**preparedata_setting)
autohf_settings = {"resources_per_trial": {"gpu": 1, "cpu": 1},
- "num_samples": -1, # unlimited sample size
+ "num_samples": -1, # unlimited sample size
"time_budget": 3600,
"ckpt_per_epoch": 1,
"fp16": False,
}
-validation_metric, analysis = \
- autohf.fit(**autohf_settings,)
+validation_metric, analysis = autohf.fit(**autohf_settings)
```
The current use cases that are supported:
+
1. A simplified version of fine-tuning the GLUE dataset using HuggingFace;
2. For selecting better search space for fine-tuning the GLUE dataset;
-3. Use the search algorithms in flaml for more efficient fine-tuning of HuggingFace;
+3. Use the search algorithms in flaml for more efficient fine-tuning of HuggingFace.
The use cases that can be supported in future:
+
1. HPO fine-tuning for text generation;
-2. HPO fine-tuning for question answering;
+2. HPO fine-tuning for question answering.
-### Troubleshooting fine-tuning HPO for pre-trained language models
+## Troubleshooting fine-tuning HPO for pre-trained language models
To reproduce the results for our ACL2021 paper:
-*[An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. To appear in ACL-IJCNLP 2021*
+* [An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. ACL-IJCNLP 2021.
+
+```bibtex
+@inproceedings{liu2021hpo,
+ title={An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models},
+ author={Xueqing Liu and Chi Wang},
+ year={2021},
+ booktitle={ACL-IJCNLP},
+}
+```
Please refer to the following jupyter notebook: [Troubleshooting HPO for fine-tuning pre-trained language models](https://github.com/microsoft/FLAML/blob/main/notebook/research/acl2021.ipynb)
\ No newline at end of file
diff --git a/flaml/nlp/autotransformers.py b/flaml/nlp/autotransformers.py
index 803290e40..2233cc39c 100644
--- a/flaml/nlp/autotransformers.py
+++ b/flaml/nlp/autotransformers.py
@@ -455,6 +455,9 @@ class AutoTransformers:
def _get_search_algo(self,
search_algo_name,
search_algo_args_mode,
+ time_budget,
+ metric_name,
+ metric_mode_name,
**custom_hpo_args):
from .hpo.searchalgo_auto import AutoSearchAlgorithm
@@ -464,6 +467,9 @@ class AutoTransformers:
search_algo_name,
search_algo_args_mode,
self._search_space_hpo,
+ time_budget,
+ metric_name,
+ metric_mode_name,
**custom_hpo_args)
return search_algo
@@ -745,7 +751,12 @@ class AutoTransformers:
ray.init(local_mode=ray_local_mode)
self._set_search_space(**custom_hpo_args)
- search_algo = self._get_search_algo(self.jobid_config.alg, self.jobid_config.arg, **custom_hpo_args)
+ search_algo = self._get_search_algo(self.jobid_config.alg,
+ self.jobid_config.arg,
+ time_budget,
+ self.metric_name,
+ self.metric_mode_name,
+ **custom_hpo_args)
scheduler = AutoScheduler.from_scheduler_name(self.jobid_config.pru)
self.ckpt_per_epoch = ckpt_per_epoch
self.path_utils.make_dir_per_run()
diff --git a/flaml/nlp/hpo/searchalgo_auto.py b/flaml/nlp/hpo/searchalgo_auto.py
index 56ca7086f..1895c880d 100644
--- a/flaml/nlp/hpo/searchalgo_auto.py
+++ b/flaml/nlp/hpo/searchalgo_auto.py
@@ -35,7 +35,14 @@ class AutoSearchAlgorithm:
)
@classmethod
- def from_method_name(cls, search_algo_name, search_algo_args_mode, hpo_search_space, **custom_hpo_args):
+ def from_method_name(cls,
+ search_algo_name,
+ search_algo_args_mode,
+ hpo_search_space,
+ time_budget,
+ metric_name,
+ metric_mode_name,
+ **custom_hpo_args):
"""
Instantiating one of the search algorithm classes based on the search algorithm name, search algorithm
argument mode, hpo search space and other keyword args
@@ -85,15 +92,26 @@ class AutoSearchAlgorithm:
"""
if search_algo_args_mode == "dft":
this_search_algo_kwargs = DEFAULT_SEARCH_ALGO_ARGS_MAPPING[search_algo_name](
- "dft", hpo_search_space=hpo_search_space, **allowed_custom_args)
+ "dft",
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=hpo_search_space,
+ **allowed_custom_args)
elif search_algo_args_mode == "cus":
this_search_algo_kwargs = DEFAULT_SEARCH_ALGO_ARGS_MAPPING[search_algo_name](
- "cus", hpo_search_space=hpo_search_space, **allowed_custom_args)
+ "cus",
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=hpo_search_space,
+ **allowed_custom_args)
"""
returning the hpo algorithm with the arguments
"""
- return SEARCH_ALGO_MAPPING[search_algo_name](**this_search_algo_kwargs)
+ search_algo = SEARCH_ALGO_MAPPING[search_algo_name](**this_search_algo_kwargs)
+ if search_algo_name == "bs":
+ search_algo.set_search_properties(config={"time_budget_s": time_budget})
+ return search_algo
raise ValueError(
"Unrecognized method {} for this kind of AutoSearchAlgorithm: {}.\n"
"Method name should be one of {}.".format(
@@ -109,11 +127,19 @@ class AutoSearchAlgorithm:
return config_list
-def get_search_algo_args_optuna(search_args_mode, hpo_search_space=None, **custom_hpo_args):
+def get_search_algo_args_optuna(search_args_mode,
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=None,
+ **custom_hpo_args):
return {}
-def default_search_algo_args_bs(search_args_mode, hpo_search_space=None, **custom_hpo_args):
+def default_search_algo_args_bs(search_args_mode,
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=None,
+ **custom_hpo_args):
assert hpo_search_space, "hpo_search_space needs to be specified for calling AutoSearchAlgorithm.from_method_name"
if "num_train_epochs" in hpo_search_space and \
isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Categorical):
@@ -126,48 +152,28 @@ def default_search_algo_args_bs(search_args_mode, hpo_search_space=None, **custo
"num_train_epochs": min_epoch,
"per_device_train_batch_size": max(hpo_search_space["per_device_train_batch_size"].categories),
},
+ "space": hpo_search_space,
+ "metric": metric_name,
+ "mode": metric_mode_name
}
if search_args_mode == "cus":
default_search_algo_args.update(custom_hpo_args)
return default_search_algo_args
-def experiment_search_algo_args_bs(hpo_search_space=None):
- if "num_train_epochs" in hpo_search_space and \
- isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Categorical):
- min_epoch = min(hpo_search_space["num_train_epochs"].categories)
- else:
- assert isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Float)
- min_epoch = hpo_search_space["num_train_epochs"].lower
- default_search_algo_args = {
- "low_cost_partial_config": {
- "num_train_epochs": min_epoch,
- },
- }
- return default_search_algo_args
-
-
-def default_search_algo_args_skopt(hpo_search_space=None):
- return {}
-
-
-def default_search_algo_args_dragonfly(hpo_search_space=None):
- return {}
-
-
-def default_search_algo_args_nevergrad(hpo_search_space=None):
- return {}
-
-
-def default_search_algo_args_hyperopt(hpo_search_space=None):
- return {}
-
-
-def default_search_algo_args_grid_search(search_args_mode, hpo_search_space=None, **custom_hpo_args):
+def default_search_algo_args_grid_search(search_args_mode,
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=None,
+ **custom_hpo_args):
return {}
-def default_search_algo_args_random_search(search_args_mode, hpo_search_space=None, **custom_hpo_args):
+def default_search_algo_args_random_search(search_args_mode,
+ metric_name,
+ metric_mode_name,
+ hpo_search_space=None,
+ **custom_hpo_args):
return {}
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py
index 280e74671..88884f982 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/searcher/blendsearch.py
@@ -12,10 +12,11 @@ try:
from ray.tune.suggest import Searcher
from ray.tune.suggest.optuna import OptunaSearch as GlobalSearch
from ray.tune.suggest.variant_generator import generate_variants
+ from ray.tune.utils.util import flatten_dict
except ImportError:
from .suggestion import Searcher
from .suggestion import OptunaSearch as GlobalSearch
- from .variant_generator import generate_variants
+ from .variant_generator import generate_variants, flatten_dict
from .search_thread import SearchThread
from .flow2 import FLOW2
@@ -48,7 +49,8 @@ class BlendSearch(Searcher):
List[Tuple[Callable[[dict], float], str, float]]] = None,
metric_constraints: Optional[
List[Tuple[str, str, float]]] = None,
- seed: Optional[int] = 20):
+ seed: Optional[int] = 20,
+ experimental: Optional[bool] = False):
'''Constructor
Args:
@@ -106,6 +108,7 @@ class BlendSearch(Searcher):
metric_constraints: A list of metric constraints to be satisfied.
e.g., `['precision', '>=', 0.9]`
seed: An integer of the random seed.
+ experimental: A bool of whether to use experimental features.
'''
self._metric, self._mode = metric, mode
init_config = low_cost_partial_config or {}
@@ -127,11 +130,20 @@ class BlendSearch(Searcher):
elif getattr(self, '__name__', None) != 'CFO':
try:
gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)
- self._gs = GlobalSearch(space=space, metric=metric, mode=mode, seed=gs_seed)
+ if experimental:
+ import optuna as ot
+ sampler = ot.samplers.TPESampler(
+ seed=seed, multivariate=True, group=True)
+ else:
+ sampler = None
+ self._gs = GlobalSearch(
+ space=space, metric=metric, mode=mode, seed=gs_seed,
+ sampler=sampler)
except TypeError:
self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
+ self._experimental = experimental
if getattr(self, '__name__', None) == 'CFO' and points_to_evaluate and len(
points_to_evaluate) > 1:
# use the best config in points_to_evaluate as the start point
@@ -292,8 +304,14 @@ class BlendSearch(Searcher):
objective = result[self._ls.metric]
if (objective - self._metric_target) * self._ls.metric_op < 0:
self._metric_target = objective
- if thread_id == 0 and metric_constraint_satisfied \
- and self._create_condition(result):
+ if thread_id:
+ if not self._metric_constraint_satisfied:
+ # no point has been found to satisfy metric constraint
+ self._expand_admissible_region()
+ if self._gs is not None and self._experimental:
+ self._gs.add_evaluated_point(flatten_dict(config), objective)
+ elif metric_constraint_satisfied and self._create_condition(
+ result):
# thread creator
thread_id = self._thread_count
self._started_from_given = self._candidate_start_points \
@@ -303,9 +321,6 @@ class BlendSearch(Searcher):
else:
self._started_from_low_cost = True
self._create_thread(config, result)
- elif thread_id and not self._metric_constraint_satisfied:
- # no point has been found to satisfy metric constraint
- self._expand_admissible_region()
# reset admissible region to ls bounding box
self._gs_admissible_min.update(self._ls_bound_min)
self._gs_admissible_max.update(self._ls_bound_max)
diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.py
index faa6fdc3f..b807e6fe2 100644
--- a/flaml/searcher/flow2.py
+++ b/flaml/searcher/flow2.py
@@ -290,9 +290,12 @@ class FLOW2(Searcher):
return unflatten_dict(config)
def create(self, init_config: Dict, obj: float, cost: float) -> Searcher:
+ flatten_config = flatten_dict(init_config)
+ # use the subspace where the init_config is located
+ space = {k: self.space[k] for k in flatten_config if k in self.space}
flow2 = self.__class__(
init_config, self.metric, self.mode, self._cat_hp_cost,
- unflatten_dict(self.space), self.prune_attr,
+ unflatten_dict(space), self.prune_attr,
self.min_resource, self.max_resource,
self.resource_multiple_factor, self.cost_attr, self._seed + 1)
flow2.best_obj = obj * self.metric_op # minimize internally
diff --git a/flaml/tune/README.md b/flaml/tune/README.md
index 506a92e9b..aada353da 100644
--- a/flaml/tune/README.md
+++ b/flaml/tune/README.md
@@ -71,16 +71,20 @@ config_search_space = {
low_cost_partial_config={'x':1}
# set up CFO
-search_alg_cfo = CFO(low_cost_partial_config=low_cost_partial_config)
-
-# set up BlendSearch.
-search_alg_blendsearch = BlendSearch(metric="metric",
- mode="min",
- space=config_search_space,
- low_cost_partial_config=low_cost_partial_config)
-# NOTE that when using BlendSearch as a search_alg in ray tune, you need to
-# configure the 'time_budget_s' for BlendSearch accordingly as follows such that BlendSearch is aware of the time budget. This step is not needed when BlendSearch is used as the search_alg in flaml.tune as it is already done automatically in flaml.
-search_alg_blendsearch.set_search_properties(config={"time_budget_s": time_budget_s})
+cfo = CFO(low_cost_partial_config=low_cost_partial_config)
+
+# set up BlendSearch
+blendsearch = BlendSearch(
+ metric="metric", mode="min",
+ space=config_search_space,
+ low_cost_partial_config=low_cost_partial_config)
+# NOTE: when using BlendSearch as a search_alg in ray tune, you need to
+# configure the 'time_budget_s' for BlendSearch accordingly as follows such that
+# BlendSearch is aware of the time budget. This step is not needed when
+# BlendSearch is used as the search_alg in flaml.tune as it is already done
+# automatically in flaml. Also, this step needs to be done after the search
+# space is passed to BlendSearch and before raytune.run.
+blendsearch.set_search_properties(config={"time_budget_s": time_budget_s})
analysis = raytune.run(
evaluate_config, # the function to evaluate a config
@@ -90,11 +94,11 @@ analysis = raytune.run(
num_samples=-1, # the maximal number of configs to try, -1 means infinite
time_budget_s=time_budget_s, # the time budget in seconds
local_dir='logs/', # the local directory to store logs
- search_alg=search_alg_blendsearch # or search_alg_cfo
+ search_alg=blendsearch # or cfo
)
print(analysis.best_trial.last_result) # the best trial's result
-print(analysis.best_config) # the best config
+print(analysis.best_config) # the best config
```
* Example for using NNI: An example of using BlendSearch with NNI can be seen in [test](https://github.com/microsoft/FLAML/tree/main/test/nni). CFO can be used as well in a similar manner. To run the example, first make sure you have [NNI](https://nni.readthedocs.io/en/stable/) installed, then run:
diff --git a/flaml/tune/cgmanifest.json b/flaml/tune/cgmanifest.json
index 425df4716..96626929b 100644
--- a/flaml/tune/cgmanifest.json
+++ b/flaml/tune/cgmanifest.json
@@ -3,7 +3,7 @@
{
"Component": {
"Type": "pip",
- "pip": {"Name": "ray[tune]", "Version": "1.2.0" }
+ "pip": {"Name": "ray[tune]", "Version": "1.5.1" }
},
"DevelopmentDependency": false
},
diff --git a/flaml/tune/space.py b/flaml/tune/space.py
new file mode 100644
index 000000000..0ffbedac1
--- /dev/null
+++ b/flaml/tune/space.py
@@ -0,0 +1,78 @@
+
+try:
+ from ray.tune import sample
+except ImportError:
+ from . import sample
+from typing import Dict, Optional, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def define_by_run_func(
+ trial, space: Dict, path: str = ""
+) -> Optional[Dict[str, Any]]:
+ """Define-by-run function to create the search space.
+
+ Returns:
+ None or a dict with constant values.
+ """
+ config = {}
+ for key, domain in space.items():
+ if path:
+ key = path + '/' + key
+ if not isinstance(domain, sample.Domain):
+ config[key] = domain
+ continue
+ sampler = domain.get_sampler()
+ quantize = None
+ if isinstance(sampler, sample.Quantized):
+ quantize = sampler.q
+ sampler = sampler.sampler
+ if isinstance(sampler, sample.LogUniform):
+ logger.warning(
+ "Optuna does not handle quantization in loguniform "
+ "sampling. The parameter will be passed but it will "
+ "probably be ignored.")
+ if isinstance(domain, sample.Float):
+ if isinstance(sampler, sample.LogUniform):
+ if quantize:
+ logger.warning(
+ "Optuna does not support both quantization and "
+ "sampling from LogUniform. Dropped quantization.")
+ trial.suggest_float(
+ key, domain.lower, domain.upper, log=True)
+ elif isinstance(sampler, sample.Uniform):
+ if quantize:
+ trial.suggest_float(
+ key, domain.lower, domain.upper, step=quantize)
+ trial.suggest_float(key, domain.lower, domain.upper)
+ elif isinstance(domain, sample.Integer):
+ if isinstance(sampler, sample.LogUniform):
+ trial.suggest_int(
+ key, domain.lower, domain.upper, step=quantize or 1, log=True)
+ elif isinstance(sampler, sample.Uniform):
+ # Upper bound should be inclusive for quantization and
+ # exclusive otherwise
+ trial.suggest_int(
+ key, domain.lower, domain.upper, step=quantize or 1)
+ elif isinstance(domain, sample.Categorical):
+ if isinstance(sampler, sample.Uniform):
+ if not hasattr(domain, 'choices'):
+ domain.choices = list(range(len(domain.categories)))
+ choices = domain.choices
+ # This choice needs to be removed from the final config
+ index = trial.suggest_categorical(key + '_choice_', choices)
+ choice = domain.categories[index]
+ if isinstance(choice, dict):
+ key += f":{index}"
+ # the suffix needs to be removed from the final config
+ config[key] = define_by_run_func(trial, choice, key)
+ else:
+ raise ValueError(
+ "Optuna search does not support parameters of type "
+ "`{}` with samplers of type `{}`".format(
+ type(domain).__name__,
+ type(domain.sampler).__name__))
+ # Return all constants in a dictionary.
+ return config
diff --git a/notebook/research/acl2021.ipynb b/notebook/research/acl2021.ipynb
index 18aef2b99..13d48846e 100644
--- a/notebook/research/acl2021.ipynb
+++ b/notebook/research/acl2021.ipynb
@@ -4,30 +4,31 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Copyright (c) 2020-2021. All rights reserved.\n",
- "\n",
- "Licensed under the MIT License.\n",
- "\n",
- "# Troubleshooting HPO for fine-tuning pre-trained language models\n",
- "\n",
- "## 1. Introduction\n",
- "\n",
- "\n",
- "In this notebook, we demonstrate a procedure for troubleshooting HPO failure in fine-tuning pre-trained language models (introduced in the following paper):\n",
- "\n",
- "*[An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. To appear in ACL-IJCNLP 2021*\n",
- "\n",
- "Notes:\n",
- "\n",
- "*In this notebook, we only run each experiment 1 time for simplicity, which is different from the paper (3 times). To reproduce the paper's result, please run 3 repetitions and take the average scores.\n",
- "\n",
- "*Running this notebook takes about one hour.\n",
- "\n",
- "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` and `nlp` options:\n",
- "```bash\n",
- "pip install flaml[nlp]\n",
- "```\n",
- "Our paper was developed under transformers version 3.4.0. We uninstall and reinstall transformers==3.4.0:"
+ "Copyright (c) 2020-2021. All rights reserved.\r\n",
+ "\r\n",
+ "Licensed under the MIT License.\r\n",
+ "\r\n",
+ "# Troubleshooting HPO for fine-tuning pre-trained language models\r\n",
+ "\r\n",
+ "## 1. Introduction\r\n",
+ "\r\n",
+ "In this notebook, we demonstrate a procedure for troubleshooting HPO failure in fine-tuning pre-trained language models (introduced in the following paper):\r\n",
+ "\r\n",
+ "*[An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. ACL-IJCNLP 2021*\r\n",
+ "\r\n",
+ "Notes:\r\n",
+ "\r\n",
+ "*In this notebook, we only run each experiment 1 time for simplicity, which is different from the paper (3 times). To reproduce the paper's result, please run 3 repetitions and take the average scores.\r\n",
+ "\r\n",
+ "*Running this notebook takes about one hour.\r\n",
+ "\r\n",
+ "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` and `nlp` options:\r\n",
+ "\r\n",
+ "```bash\r\n",
+ "pip install flaml[nlp]\r\n",
+ "```\r\n",
+ "\r\n",
+ "Our paper was developed under transformers version 3.4.0. We uninstall and reinstall transformers==3.4.0:\r\n"
]
},
{
@@ -40,9 +41,9 @@
},
"outputs": [],
"source": [
- "!pip install flaml[nlp]\n",
- "!pip install transformers==3.4.0\n",
- "from flaml.nlp import AutoTransformers\n"
+ "!pip install flaml[nlp]\r\n",
+ "!pip install transformers==3.4.0\r\n",
+ "from flaml.nlp import AutoTransformers\r\n"
]
},
{
@@ -363,10 +364,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "\u001B[2m\u001B[36m(pid=50964)\u001B[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
- "\u001B[2m\u001B[36m(pid=50964)\u001B[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
- "\u001B[2m\u001B[36m(pid=50948)\u001B[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n",
- "\u001B[2m\u001B[36m(pid=50948)\u001B[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n"
+ "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
+ "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
+ "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n",
+ "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n"
]
},
{
@@ -484,12 +485,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n"
+ "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n"
]
},
{
@@ -589,18 +590,18 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n"
+ "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
+ "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
+ "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
+ "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
+ "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
+ "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
+ "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
+ "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
+ "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n"
]
},
{
@@ -700,21 +701,21 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n"
+ "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
+ "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
+ "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
+ "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
+ "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
+ "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
+ "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
+ "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
+ "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
+ "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
+ "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n"
]
},
{
@@ -791,22 +792,16 @@
}
],
"metadata": {
+ "interpreter": {
+ "hash": "bfcd9a6a9254a5e160761a1fd7a9e444f011592c6770d9f4180dde058a9df5dd"
+ },
"kernelspec": {
- "display_name": "Python 3",
- "language": "python",
+ "display_name": "Python 3.7.7 64-bit ('flaml': conda)",
"name": "python3"
},
"language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
"name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.0"
+ "version": ""
}
},
"nbformat": 4,
diff --git a/setup.py b/setup.py
index 67a87345a..14d858e79 100644
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,7 @@ setuptools.setup(
"optuna==2.8.0"
],
"ray": [
- "ray[tune]==1.4.1",
+ "ray[tune]==1.5.1",
"pyyaml<5.3.1",
],
"azureml": [
@@ -74,7 +74,7 @@ setuptools.setup(
"vowpalwabbit",
],
"nlp": [
- "ray[tune]>=1.4.1",
+ "ray[tune]>=1.5.1",
"transformers",
"datasets==1.4.1",
"tensorboardX<=2.2",
diff --git a/test/test_notebook_example.py b/test/test_notebook_example.py
index 286786daf..a669a65cb 100644
--- a/test/test_notebook_example.py
+++ b/test/test_notebook_example.py
@@ -37,6 +37,9 @@ def test_automl(budget=5):
get_output_from_log(filename=settings['log_file_name'], time_budget=60)
for config in config_history:
print(config)
+ print(automl.prune_attr)
+ print(automl.max_resource)
+ print(automl.min_resource)
def test_mlflow():
diff --git a/test/test_python_log.py b/test/test_python_log.py
index 88995d4f7..732c881d1 100644
--- a/test/test_python_log.py
+++ b/test/test_python_log.py
@@ -42,7 +42,16 @@ class TestLogging(unittest.TestCase):
automl.fit(X_train=X_train[:n], y_train=y_train[:n],
X_val=X_train[n:], y_val=y_train[n:],
**automl_settings)
-
+ logger.info(automl.search_space)
+ logger.info(automl.low_cost_partial_config)
+ logger.info(automl.points_to_evalaute)
+ import optuna as ot
+ study = ot.create_study()
+ from flaml.tune.space import define_by_run_func
+ logger.info(define_by_run_func(study.ask(), automl.search_space))
+ config = automl.best_config.copy()
+ config['learner'] = automl.best_estimator
+ automl.trainable({"ml": config})
# Check if the log buffer is populated.
self.assertTrue(len(buf.getvalue()) > 0)
diff --git a/test/test_xgboost2d.py b/test/test_xgboost2d.py
index 562401684..c9f380c67 100644
--- a/test/test_xgboost2d.py
+++ b/test/test_xgboost2d.py
@@ -19,11 +19,11 @@ class XGBoost2D(XGBoostSklearnEstimator):
return {
'n_estimators': {
'domain': tune.lograndint(lower=4, upper=upper),
- 'init_value': 4,
+ 'low_cost_init_value': 4,
},
'max_leaves': {
'domain': tune.lograndint(lower=4, upper=upper),
- 'init_value': 4,
+ 'low_cost_init_value': 4,
},
}
@@ -40,7 +40,7 @@ def test_simple(method=None):
"n_jobs": 1,
"hpo_method": method,
"log_type": "all",
- "time_budget": 3
+ "time_budget": 1
}
from sklearn.externals._arff import ArffException
try:
@@ -51,6 +51,25 @@ def test_simple(method=None):
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.33, random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
+ print(automl.estimator_list)
+ print(automl.search_space)
+ print(automl.points_to_evalaute)
+ config = automl.best_config.copy()
+ config['learner'] = automl.best_estimator
+ automl.trainable(config)
+ from flaml import tune
+ analysis = tune.run(
+ automl.trainable, automl.search_space, metric='val_loss',
+ low_cost_partial_config=automl.low_cost_partial_config,
+ points_to_evaluate=automl.points_to_evalaute,
+ cat_hp_cost=automl.cat_hp_cost,
+ prune_attr=automl.prune_attr,
+ min_resource=automl.min_resource,
+ max_resource=automl.max_resource,
+ time_budget_s=automl._state.time_budget,
+ config_constraints=[(automl.size, '<=', automl._mem_thres)],
+ metric_constraints=automl.metric_constraints)
+ print(analysis.trials[-1])
def _test_optuna():
diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py
index 8ab7d4e06..1d0f96032 100644
--- a/test/tune/test_tune.py
+++ b/test/tune/test_tune.py
@@ -1,5 +1,6 @@
'''Require: pip install flaml[test,ray]
'''
+from flaml.searcher.blendsearch import BlendSearch
import time
import os
from sklearn.model_selection import train_test_split
@@ -199,6 +200,28 @@ def test_nested():
logger.info(f"CFO best config: {best_trial.config}")
logger.info(f"CFO best result: {best_trial.last_result}")
+ analysis = tune.run(
+ simple_func,
+ search_alg=BlendSearch(
+ experimental=True,
+ space=search_space, metric="obj", mode="min",
+ low_cost_partial_config={
+ "cost_related": {"a": 1}
+ },
+ points_to_evaluate=[
+ {"b": .99, "cost_related": {"a": 3}},
+ {"b": .99, "cost_related": {"a": 2}},
+ {"cost_related": {"a": 8}}
+ ],
+ metric_constraints=[("ab", "<=", 4)]),
+ local_dir='logs/',
+ num_samples=-1,
+ time_budget_s=.1)
+
+ best_trial = analysis.get_best_trial()
+ logger.info(f"BlendSearch exp best config: {best_trial.config}")
+ logger.info(f"BlendSearch exp best result: {best_trial.last_result}")
+
analysis = tune.run(
simple_func,
config=search_space,
@@ -222,7 +245,7 @@ def test_run_training_function_return_value():
# Test dict return value
def evaluate_config_dict(config):
- metric = (round(config['x'])-85000)**2 - config['x']/config['y']
+ metric = (round(config['x']) - 85000)**2 - config['x'] / config['y']
return {"metric": metric}
tune.run(
@@ -236,7 +259,7 @@ def test_run_training_function_return_value():
# Test scalar return value
def evaluate_config_scalar(config):
- metric = (round(config['x'])-85000)**2 - config['x']/config['y']
+ metric = (round(config['x']) - 85000)**2 - config['x'] / config['y']
return metric
tune.run(