A programming framework for agentic AI
consider num_samples in bs thread priority (#207)
Chi Wanga9d39b71dae7
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex d9d02a898..d03fe4b56 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -1,9 +1,9 @@-'''!- * Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved.+"""!+ * Copyright (c) Microsoft Corporation. All rights reserved.* Licensed under the MIT License. See LICENSE file in the* project root for license information.-'''-from typing import Dict, Optional, List, Tuple, Callable+"""+from typing import Dict, Optional, List, Tuple, Callable, Unionimport numpy as npimport timeimport pickle@@ -11,7 +11,8 @@ import pickletry:from ray import __version__ as ray_version- assert ray_version >= '1.0.0'++ assert ray_version >= "1.0.0"from ray.tune.suggest import Searcherfrom ray.tune.suggest.optuna import OptunaSearch as GlobalSearchexcept (ImportError, AssertionError):@@ -20,42 +21,45 @@ except (ImportError, AssertionError):from ..tune.trial import unflatten_dict, flatten_dictfrom .search_thread import SearchThreadfrom .flow2 import FLOW2-from ..tune.space import (- add_cost_to_space, indexof, normalize, define_by_run_func)+from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_funcimport logging+logger = logging.getLogger(__name__)class BlendSearch(Searcher):- '''class for BlendSearch algorithm- '''+ """class for BlendSearch algorithm"""cost_attr = "time_total_s" # cost attribute in result- lagrange = '_lagrange' # suffix for lagrange-modified metric- penalty = 1e+10 # penalty term for constraints+ lagrange = "_lagrange" # suffix for lagrange-modified metric+ penalty = 1e10 # penalty term for constraintsLocalSearch = FLOW2- def __init__(self,- metric: Optional[str] = None,- mode: Optional[str] = None,- space: Optional[dict] = None,- low_cost_partial_config: Optional[dict] = None,- cat_hp_cost: Optional[dict] = None,- points_to_evaluate: Optional[List[dict]] = None,- evaluated_rewards: Optional[List] = None,- prune_attr: Optional[str] = None,- min_resource: Optional[float] = None,- max_resource: Optional[float] = None,- reduction_factor: Optional[float] = None,- global_search_alg: Optional[Searcher] = None,- config_constraints: Optional[- List[Tuple[Callable[[dict], float], str, float]]] = None,- metric_constraints: Optional[- List[Tuple[str, str, float]]] = None,- seed: Optional[int] = 20,- experimental: Optional[bool] = False):- '''Constructor+ def __init__(+ self,+ metric: Optional[str] = None,+ mode: Optional[str] = None,+ space: Optional[dict] = None,+ low_cost_partial_config: Optional[dict] = None,+ cat_hp_cost: Optional[dict] = None,+ points_to_evaluate: Optional[List[dict]] = None,+ evaluated_rewards: Optional[List] = None,+ time_budget_s: Union[int, float] = None,+ num_samples: Optional[int] = None,+ prune_attr: Optional[str] = None,+ min_resource: Optional[float] = None,+ max_resource: Optional[float] = None,+ reduction_factor: Optional[float] = None,+ global_search_alg: Optional[Searcher] = None,+ config_constraints: Optional[+ List[Tuple[Callable[[dict], float], str, float]]+ ] = None,+ metric_constraints: Optional[List[Tuple[str, str, float]]] = None,+ seed: Optional[int] = 20,+ experimental: Optional[bool] = False,+ ):+ """ConstructorArgs:metric: A string of the metric name to optimize for.@@ -87,6 +91,8 @@ class BlendSearch(Searcher):as a list so the optimiser can be told the results withoutneeding to re-compute the trial. Must be the same length aspoints_to_evaluate.+ time_budget_s: int or float | Time budget in seconds.+ num_samples: int | The number of configs to try.prune_attr: A string of the attribute used for pruning.Not necessarily in space.When prune_attr is in space, it is a hyperparameter, e.g.,@@ -119,7 +125,7 @@ class BlendSearch(Searcher):e.g., `['precision', '>=', 0.9]`seed: An integer of the random seed.experimental: A bool of whether to use experimental features.- '''+ """self._metric, self._mode = metric, modeinit_config = low_cost_partial_config or {}if not init_config:@@ -132,8 +138,7 @@ class BlendSearch(Searcher):if evaluated_rewards and mode:self._points_to_evaluate = []self._evaluated_rewards = []- best = max(evaluated_rewards) if mode == 'max' else min(- evaluated_rewards)+ best = max(evaluated_rewards) if mode == "max" else min(evaluated_rewards)# only keep the best points as start pointsfor i, r in enumerate(evaluated_rewards):if r == best:@@ -152,51 +157,78 @@ class BlendSearch(Searcher):if space:add_cost_to_space(space, init_config, self._cat_hp_cost)self._ls = self.LocalSearch(- init_config, metric, mode, space, prune_attr,- min_resource, max_resource, reduction_factor, self.cost_attr, seed)+ init_config,+ metric,+ mode,+ space,+ prune_attr,+ min_resource,+ max_resource,+ reduction_factor,+ self.cost_attr,+ seed,+ )if global_search_alg is not None:self._gs = global_search_alg- elif getattr(self, '__name__', None) != 'CFO':+ elif getattr(self, "__name__", None) != "CFO":if space and self._ls.hierarchical:from functools import partial+gs_space = partial(define_by_run_func, space=space)- evaluated_rewards = None # not supproted by define-by-run+ evaluated_rewards = None # not supproted by define-by-runelse:gs_space = spacegs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)if experimental:import optuna as ot+sampler = ot.samplers.TPESampler(- seed=seed, multivariate=True, group=True)+ seed=seed, multivariate=True, group=True+ )else:sampler = Nonetry:self._gs = GlobalSearch(- space=gs_space, metric=metric, mode=mode, seed=gs_seed,- sampler=sampler, points_to_evaluate=points_to_evaluate,- evaluated_rewards=evaluated_rewards)+ space=gs_space,+ metric=metric,+ mode=mode,+ seed=gs_seed,+ sampler=sampler,+ points_to_evaluate=points_to_evaluate,+ evaluated_rewards=evaluated_rewards,+ )except ValueError:self._gs = GlobalSearch(- space=gs_space, metric=metric, mode=mode, seed=gs_seed,- sampler=sampler)+ space=gs_space,+ metric=metric,+ mode=mode,+ seed=gs_seed,+ sampler=sampler,+ )self._gs.space = spaceelse:self._gs = Noneself._experimental = experimental- if getattr(self, '__name__', None) == 'CFO' and points_to_evaluate and len(- self._points_to_evaluate) > 1:+ if (+ getattr(self, "__name__", None) == "CFO"+ and points_to_evaluate+ and len(self._points_to_evaluate) > 1+ ):# use the best config in points_to_evaluate as the start pointself._candidate_start_points = {}self._started_from_low_cost = not low_cost_partial_configelse:self._candidate_start_points = None+ self._time_budget_s, self._num_samples = time_budget_s, num_samplesif space:self._init_search()- def set_search_properties(self,- metric: Optional[str] = None,- mode: Optional[str] = None,- config: Optional[Dict] = None) -> bool:+ def set_search_properties(+ self,+ metric: Optional[str] = None,+ mode: Optional[str] = None,+ config: Optional[Dict] = None,+ ) -> bool:metric_changed = mode_changed = Falseif metric and self._metric != metric:metric_changed = True@@ -216,34 +248,54 @@ class BlendSearch(Searcher):self._gs.set_search_properties(metric, mode, config)self._gs.space = configif config:- add_cost_to_space(- config, self._ls.init_config, self._cat_hp_cost)+ add_cost_to_space(config, self._ls.init_config, self._cat_hp_cost)self._ls.set_search_properties(metric, mode, config)self._init_search()- elif metric_changed or mode_changed:- # reset search when metric or mode changed- self._ls.set_search_properties(metric, mode)- if self._gs is not None:- self._gs = GlobalSearch(- space=self._gs._space, metric=metric, mode=mode,- sampler=self._gs._sampler)- self._gs.space = self._ls.space- self._init_search()- if config:- if 'time_budget_s' in config:- time_budget_s = config['time_budget_s']- if time_budget_s is not None:- self._deadline = time_budget_s + time.time()- SearchThread.set_eps(time_budget_s)- if 'metric_target' in config:- self._metric_target = config.get('metric_target')+ else:+ if metric_changed or mode_changed:+ # reset search when metric or mode changed+ self._ls.set_search_properties(metric, mode)+ if self._gs is not None:+ self._gs = GlobalSearch(+ space=self._gs._space,+ metric=metric,+ mode=mode,+ sampler=self._gs._sampler,+ )+ self._gs.space = self._ls.space+ self._init_search()+ if config:+ # CFO doesn't need these settings+ if "time_budget_s" in config:+ self._time_budget_s = config["time_budget_s"] # budget from now+ now = time.time()+ self._time_used += now - self._start_time+ self._start_time = now+ self._set_deadline()+ if "metric_target" in config:+ self._metric_target = config.get("metric_target")+ if "num_samples" in config:+ self._num_samples = (+ config["num_samples"]+ + len(self._result)+ + len(self._trial_proposed_by)+ )return True+ def _set_deadline(self):+ if self._time_budget_s is not None:+ self._deadline = self._time_budget_s + self._start_time+ SearchThread.set_eps(self._time_budget_s)+ else:+ self._deadline = np.inf+def _init_search(self):- '''initialize the search- '''+ """initialize the search"""+ self._start_time = time.time()+ self._time_used = 0+ self._set_deadline()self._is_ls_ever_converged = False- self._subspace = {} # the subspace for each trial id+ self._subspace = {} # the subspace for each trial idself._metric_target = np.inf * self._ls.metric_opself._search_thread_pool = {# id: int -> thread: SearchThread@@ -253,35 +305,41 @@ class BlendSearch(Searcher):self._init_used = self._ls.init_config is Noneself._trial_proposed_by = {} # trial_id: str -> thread_id: intself._ls_bound_min = normalize(- self._ls.init_config.copy(), self._ls.space, self._ls.init_config,- {}, recursive=True)+ self._ls.init_config.copy(),+ self._ls.space,+ self._ls.init_config,+ {},+ recursive=True,+ )self._ls_bound_max = self._ls_bound_min.copy()self._gs_admissible_min = self._ls_bound_min.copy()self._gs_admissible_max = self._ls_bound_max.copy()self._result = {} # config_signature: tuple -> result: Dict- self._deadline = np.infif self._metric_constraints:self._metric_constraint_satisfied = Falseself._metric_constraint_penalty = [- self.penalty for _ in self._metric_constraints]+ self.penalty for _ in self._metric_constraints+ ]else:self._metric_constraint_satisfied = Trueself._metric_constraint_penalty = Noneself.best_resource = self._ls.min_resourcedef save(self, checkpoint_path: str):- ''' save states to a checkpoint path- '''+ """save states to a checkpoint path"""+ self._time_used += time.time() - self._start_time+ self._start_time = time.time()save_object = selfwith open(checkpoint_path, "wb") as outputFile:pickle.dump(save_object, outputFile)def restore(self, checkpoint_path: str):- ''' restore states from checkpoint- '''+ """restore states from checkpoint"""with open(checkpoint_path, "rb") as inputFile:state = pickle.load(inputFile)self.__dict__ = state.__dict__+ self._start_time = time.time()+ self._set_deadline()@propertydef metric_target(self):@@ -291,10 +349,10 @@ class BlendSearch(Searcher):def is_ls_ever_converged(self):return self._is_ls_ever_converged- def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None,- error: bool = False):- ''' search thread updater and cleaner- '''+ def on_trial_complete(+ self, trial_id: str, result: Optional[Dict] = None, error: bool = False+ ):+ """search thread updater and cleaner"""metric_constraint_satisfied = Trueif result and not error and self._metric_constraints:# account for metric constraints if any@@ -304,12 +362,15 @@ class BlendSearch(Searcher):value = result.get(metric_constraint)if value:# sign is <= or >=- sign_op = 1 if sign == '<=' else -1+ sign_op = 1 if sign == "<=" else -1violation = (value - threshold) * sign_opif violation > 0:# add penalty term to the metric- objective += self._metric_constraint_penalty[- i] * violation * self._ls.metric_op+ objective += (+ self._metric_constraint_penalty[i]+ * violation+ * self._ls.metric_op+ )metric_constraint_satisfied = Falseif self._metric_constraint_penalty[i] < self.penalty:self._metric_constraint_penalty[i] += violation@@ -321,16 +382,18 @@ class BlendSearch(Searcher):thread_id = self._trial_proposed_by.get(trial_id)if thread_id in self._search_thread_pool:self._search_thread_pool[thread_id].on_trial_complete(- trial_id, result, error)+ trial_id, result, error+ )del self._trial_proposed_by[trial_id]if result:- config = result.get('config', {})+ config = result.get("config", {})if not config:for key, value in result.items():- if key.startswith('config/'):+ if key.startswith("config/"):config[key[7:]] = valuesignature = self._ls.config_signature(- config, self._subspace.get(trial_id, {}))+ config, self._subspace.get(trial_id, {})+ )if error: # remove from result cachedel self._result[signature]else: # add to result cache@@ -345,28 +408,34 @@ class BlendSearch(Searcher):if not self._metric_constraint_satisfied:# no point has been found to satisfy metric constraintself._expand_admissible_region(- self._ls_bound_min, self._ls_bound_max,- self._subspace.get(trial_id, self._ls.space))- if self._gs is not None and self._experimental and (- not self._ls.hierarchical):- self._gs.add_evaluated_point(- flatten_dict(config), objective)+ self._ls_bound_min,+ self._ls_bound_max,+ self._subspace.get(trial_id, self._ls.space),+ )+ if (+ self._gs is not None+ and self._experimental+ and (not self._ls.hierarchical)+ ):+ self._gs.add_evaluated_point(flatten_dict(config), objective)# TODO: recover when supported# converted = convert_key(config, self._gs.space)# logger.info(converted)# self._gs.add_evaluated_point(converted, objective)- elif metric_constraint_satisfied and self._create_condition(- result):+ elif metric_constraint_satisfied and self._create_condition(result):# thread creatorthread_id = self._thread_count- self._started_from_given = self._candidate_start_points \+ self._started_from_given = (+ self._candidate_start_pointsand trial_id in self._candidate_start_points+ )if self._started_from_given:del self._candidate_start_points[trial_id]else:self._started_from_low_cost = True- self._create_thread(config, result, self._subspace.get(- trial_id, self._ls.space))+ self._create_thread(+ config, result, self._subspace.get(trial_id, self._ls.space)+ )# reset admissible region to ls bounding boxself._gs_admissible_min.update(self._ls_bound_min)self._gs_admissible_max.update(self._ls_bound_max)@@ -374,26 +443,38 @@ class BlendSearch(Searcher):if thread_id and thread_id in self._search_thread_pool:# local search threadself._clean(thread_id)- if trial_id in self._subspace and not (self._candidate_start_points- and trial_id in self._candidate_start_points):+ if trial_id in self._subspace and not (+ self._candidate_start_points and trial_id in self._candidate_start_points+ ):del self._subspace[trial_id]def _create_thread(self, config, result, space):self._search_thread_pool[self._thread_count] = SearchThread(self._ls.mode,self._ls.create(- config, result[self._ls.metric],- cost=result.get(self.cost_attr, 1), space=space),- self.cost_attr+ config,+ result[self._ls.metric],+ cost=result.get(self.cost_attr, 1),+ space=space,+ ),+ self.cost_attr,)self._thread_count += 1self._update_admissible_region(- unflatten_dict(config), self._ls_bound_min, self._ls_bound_max, space,- self._ls.space)+ unflatten_dict(config),+ self._ls_bound_min,+ self._ls_bound_max,+ space,+ self._ls.space,+ )def _update_admissible_region(- self, config, admissible_min, admissible_max, subspace: Dict = {},- space: Dict = {}+ self,+ config,+ admissible_min,+ admissible_max,+ subspace: Dict = {},+ space: Dict = {},):# update admissible regionnormalized_config = normalize(config, subspace, config, {})@@ -404,13 +485,19 @@ class BlendSearch(Searcher):choice = indexof(domain, value)self._update_admissible_region(value,- admissible_min[key][choice], admissible_max[key][choice],- subspace[key], domain[choice]+ admissible_min[key][choice],+ admissible_max[key][choice],+ subspace[key],+ domain[choice],)elif isinstance(value, dict):self._update_admissible_region(- value, admissible_min[key], admissible_max[key],- subspace[key], space[key])+ value,+ admissible_min[key],+ admissible_max[key],+ subspace[key],+ space[key],+ )else:if value > admissible_max[key]:admissible_max[key] = value@@ -418,19 +505,18 @@ class BlendSearch(Searcher):admissible_min[key] = valuedef _create_condition(self, result: Dict) -> bool:- ''' create thread condition- '''+ """create thread condition"""if len(self._search_thread_pool) < 2:return Trueobj_median = np.median(- [thread.obj_best1 for id, thread in self._search_thread_pool.items()- if id])+ [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]+ )return result[self._ls.metric] * self._ls.metric_op < obj_mediandef _clean(self, thread_id: int):- ''' delete thread and increase admissible region if converged,+ """delete thread and increase admissible region if converged,merge local threads if they are close- '''+ """assert thread_idtodelete = set()for id in self._search_thread_pool:@@ -447,8 +533,10 @@ class BlendSearch(Searcher):self._is_ls_ever_converged = Truetodelete.add(thread_id)self._expand_admissible_region(- self._ls_bound_min, self._ls_bound_max,- self._search_thread_pool[thread_id].space)+ self._ls_bound_min,+ self._ls_bound_max,+ self._search_thread_pool[thread_id].space,+ )if self._candidate_start_points:if not self._started_from_given:# remove start points whose perf is worse than the converged@@ -456,7 +544,8 @@ class BlendSearch(Searcher):worse = [trial_idfor trial_id, r in self._candidate_start_points.items()- if r and r[self._ls.metric] * self._ls.metric_op >= obj]+ if r and r[self._ls.metric] * self._ls.metric_op >= obj+ ]# logger.info(f"remove candidate start points {worse} than {obj}")for trial_id in worse:del self._candidate_start_points[trial_id]@@ -472,8 +561,10 @@ class BlendSearch(Searcher):best_trial_id = Noneobj_best = Nonefor trial_id, r in self._candidate_start_points.items():- if r and (best_trial_id is None- or r[self._ls.metric] * self._ls.metric_op < obj_best):+ if r and (+ best_trial_id is None+ or r[self._ls.metric] * self._ls.metric_op < obj_best+ ):best_trial_id = trial_idobj_best = r[self._ls.metric] * self._ls.metric_opif best_trial_id:@@ -481,20 +572,22 @@ class BlendSearch(Searcher):config = {}result = self._candidate_start_points[best_trial_id]for key, value in result.items():- if key.startswith('config/'):+ if key.startswith("config/"):config[key[7:]] = valueself._started_from_given = Truedel self._candidate_start_points[best_trial_id]- self._create_thread(config, result, self._subspace.get(- best_trial_id, self._ls.space))+ self._create_thread(+ config, result, self._subspace.get(best_trial_id, self._ls.space)+ )def _expand_admissible_region(self, lower, upper, space):for key in upper:ub = upper[key]if isinstance(ub, list):- choice = space[key]['_choice_']+ choice = space[key]["_choice_"]self._expand_admissible_region(- lower[key][choice], upper[key][choice], space[key])+ lower[key][choice], upper[key][choice], space[key]+ )elif isinstance(ub, dict):self._expand_admissible_region(lower[key], ub, space[key])else:@@ -502,8 +595,7 @@ class BlendSearch(Searcher):lower[key] -= self._ls.STEPSIZEdef _inferior(self, id1: int, id2: int) -> bool:- ''' whether thread id1 is inferior to id2- '''+ """whether thread id1 is inferior to id2"""t1 = self._search_thread_pool[id1]t2 = self._search_thread_pool[id2]if t1.obj_best1 < t2.obj_best2:@@ -515,8 +607,7 @@ class BlendSearch(Searcher):return Falsedef on_trial_result(self, trial_id: str, result: Dict):- ''' receive intermediate result- '''+ """receive intermediate result"""if trial_id not in self._trial_proposed_by:returnthread_id = self._trial_proposed_by[trial_id]@@ -527,8 +618,7 @@ class BlendSearch(Searcher):self._search_thread_pool[thread_id].on_trial_result(trial_id, result)def suggest(self, trial_id: str) -> Optional[Dict]:- ''' choose thread, suggest a valid config- '''+ """choose thread, suggest a valid config"""if self._init_used and not self._points_to_evaluate:choice, backup = self._select_thread()# if choice < 0: # timeout@@ -540,8 +630,10 @@ class BlendSearch(Searcher):# local search thread finishesif self._search_thread_pool[choice].converged:self._expand_admissible_region(- self._ls_bound_min, self._ls_bound_max,- self._search_thread_pool[choice].space)+ self._ls_bound_min,+ self._ls_bound_max,+ self._search_thread_pool[choice].space,+ )del self._search_thread_pool[choice]return None# preliminary check; not checking config validation@@ -558,8 +650,12 @@ class BlendSearch(Searcher):return Noneuse_rs = 1if choice or self._valid(- config, self._ls.space, space, self._gs_admissible_min,- self._gs_admissible_max):+ config,+ self._ls.space,+ space,+ self._gs_admissible_min,+ self._gs_admissible_max,+ ):# LS or valid or no backup choiceself._trial_proposed_by[trial_id] = choiceself._search_thread_pool[choice].running += use_rs@@ -568,7 +664,8 @@ class BlendSearch(Searcher):# use CFO's init pointinit_config = self._ls.init_configconfig, space = self._ls.complete_config(- init_config, self._ls_bound_min, self._ls_bound_max)+ init_config, self._ls_bound_min, self._ls_bound_max+ )self._trial_proposed_by[trial_id] = choiceself._search_thread_pool[choice].running += 1else:@@ -583,12 +680,20 @@ class BlendSearch(Searcher):if not choice: # global search# temporarily relax admissible region for parallel proposalsself._update_admissible_region(- config, self._gs_admissible_min, self._gs_admissible_max,- space, self._ls.space)+ config,+ self._gs_admissible_min,+ self._gs_admissible_max,+ space,+ self._ls.space,+ )else:self._update_admissible_region(- config, self._ls_bound_min, self._ls_bound_max, space,- self._ls.space)+ config,+ self._ls_bound_min,+ self._ls_bound_max,+ space,+ self._ls.space,+ )self._gs_admissible_min.update(self._ls_bound_min)self._gs_admissible_max.update(self._ls_bound_max)signature = self._ls.config_signature(config, space)@@ -605,7 +710,8 @@ class BlendSearch(Searcher):else:init_config = self._ls.init_configconfig, space = self._ls.complete_config(- init_config, self._ls_bound_min, self._ls_bound_max)+ init_config, self._ls_bound_min, self._ls_bound_max+ )if reward is None:config_signature = self._ls.config_signature(config, space)result = self._result.get(config_signature)@@ -620,18 +726,15 @@ class BlendSearch(Searcher):self._search_thread_pool[0].running += 1self._subspace[trial_id] = spaceif reward is not None:- result = {- self._metric: reward, self.cost_attr: 1,- 'config': config- }+ result = {self._metric: reward, self.cost_attr: 1, "config": config}self.on_trial_complete(trial_id, result)return Nonereturn configdef _should_skip(self, choice, trial_id, config, space) -> bool:- ''' if config is None or config's result is known or constraints are violated- return True; o.w. return False- '''+ """if config is None or config's result is known or constraints are violated+ return True; o.w. return False+ """if config is None:return Trueconfig_signature = self._ls.config_signature(config, space)@@ -641,11 +744,15 @@ class BlendSearch(Searcher):for constraint in self._config_constraints:func, sign, threshold = constraintvalue = func(config)- if (sign == '<=' and value > threshold- or sign == '>=' and value < threshold):+ if (+ sign == "<="+ and value > threshold+ or sign == ">="+ and value < threshold+ ):self._result[config_signature] = {self._metric: np.inf * self._ls.metric_op,- 'time_total_s': 1,+ "time_total_s": 1,}exists = Truebreak@@ -654,7 +761,8 @@ class BlendSearch(Searcher):result = self._result.get(config_signature)if result: # finishedself._search_thread_pool[choice].on_trial_complete(- trial_id, result, error=False)+ trial_id, result, error=False+ )if choice:# local search threadself._clean(choice)@@ -666,14 +774,23 @@ class BlendSearch(Searcher):return Falsedef _select_thread(self) -> Tuple:- ''' thread selector; use can_suggest to check LS availability- '''+ """thread selector; use can_suggest to check LS availability"""# update priority- min_eci = self._deadline - time.time()+ now = time.time()+ min_eci = self._deadline - nowif min_eci <= 0:# return -1, -1# keep proposing new configs assuming no budget leftmin_eci = 0+ elif self._num_samples and self._num_samples > 0:+ # estimate time left according to num_samples limitation+ num_finished = len(self._result)+ num_proposed = num_finished + len(self._trial_proposed_by)+ num_left = max(self._num_samples - num_proposed, 0)+ if num_proposed > 0:+ time_used = now - self._start_time + self._time_used+ min_eci = min(min_eci, time_used / num_finished * num_left)+ # print(f"{min_eci}, {time_used / num_finished * num_left}, {num_finished}, {num_left}")max_speed = 0for thread in self._search_thread_pool.values():if thread.speed > max_speed:@@ -698,10 +815,10 @@ class BlendSearch(Searcher):backup_thread_id = thread_idreturn top_thread_id, backup_thread_id- def _valid(self, config: Dict, space: Dict, subspace: Dict,- lower: Dict, upper: Dict) -> bool:- ''' config validator- '''+ def _valid(+ self, config: Dict, space: Dict, subspace: Dict, lower: Dict, upper: Dict+ ) -> bool:+ """config validator"""normalized_config = normalize(config, subspace, config, {})for key, lb in lower.items():if key in config:@@ -719,114 +836,143 @@ class BlendSearch(Searcher):else:nestedspace = Noneif nestedspace:- valid = self._valid(- value, domain, nestedspace, lb, ub)+ valid = self._valid(value, domain, nestedspace, lb, ub)if not valid:return False- elif (value + self._ls.STEPSIZE < lower[key]- or value > upper[key] + self._ls.STEPSIZE):+ elif (+ value + self._ls.STEPSIZE < lower[key]+ or value > upper[key] + self._ls.STEPSIZE+ ):return Falsereturn Truetry:from ray import __version__ as ray_version- assert ray_version >= '1.0.0'- from ray.tune import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform)++ assert ray_version >= "1.0.0"+ from ray.tune import (+ uniform,+ quniform,+ choice,+ randint,+ qrandint,+ randn,+ qrandn,+ loguniform,+ qloguniform,+ )except (ImportError, AssertionError):- from ..tune.sample import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform)+ from ..tune.sample import (+ uniform,+ quniform,+ choice,+ randint,+ qrandint,+ randn,+ qrandn,+ loguniform,+ qloguniform,+ )try:from nni.tuner import Tuner as NNITunerfrom nni.utils import extract_scalar_rewardexcept ImportError:+class NNITuner:passdef extract_scalar_reward(x: Dict):- return x.get('reward')+ return x.get("reward")class BlendSearchTuner(BlendSearch, NNITuner):- '''Tuner class for NNI- '''+ """Tuner class for NNI"""- def receive_trial_result(self, parameter_id, parameters, value,- **kwargs):- '''+ def receive_trial_result(self, parameter_id, parameters, value, **kwargs):+ """Receive trial's final result.parameter_id: intparameters: object created by 'generate_parameters()'value: final metrics of the trial, including default metric- '''+ """result = {- 'config': parameters, self._metric: extract_scalar_reward(value),- self.cost_attr: 1 if isinstance(value, float) else value.get(- self.cost_attr, value.get('sequence', 1))+ "config": parameters,+ self._metric: extract_scalar_reward(value),+ self.cost_attr: 1+ if isinstance(value, float)+ else value.get(self.cost_attr, value.get("sequence", 1))# if nni does not report training cost,# using sequence as an approximation.# if no sequence, using a constant 1}self.on_trial_complete(str(parameter_id), result)+...def generate_parameters(self, parameter_id, **kwargs) -> Dict:- '''+ """Returns a set of trial (hyper-)parameters, as a serializable objectparameter_id: int- '''+ """return self.suggest(str(parameter_id))+...def update_search_space(self, search_space):- '''+ """Tuners are advised to support updating search space at run-time.If a tuner can only set search space once before generating first hyper-parameters,it should explicitly document this behaviour.search_space: JSON object created by experiment owner- '''+ """config = {}for key, value in search_space.items():v = value.get("_value")- _type = value['_type']- if _type == 'choice':+ _type = value["_type"]+ if _type == "choice":config[key] = choice(v)- elif _type == 'randint':+ elif _type == "randint":config[key] = randint(*v)- elif _type == 'uniform':+ elif _type == "uniform":config[key] = uniform(*v)- elif _type == 'quniform':+ elif _type == "quniform":config[key] = quniform(*v)- elif _type == 'loguniform':+ elif _type == "loguniform":config[key] = loguniform(*v)- elif _type == 'qloguniform':+ elif _type == "qloguniform":config[key] = qloguniform(*v)- elif _type == 'normal':+ elif _type == "normal":config[key] = randn(*v)- elif _type == 'qnormal':+ elif _type == "qnormal":config[key] = qrandn(*v)else:- raise ValueError(- f'unsupported type in search_space {_type}')+ raise ValueError(f"unsupported type in search_space {_type}")add_cost_to_space(config, {}, {})self._ls = self.LocalSearch(- {}, self._ls.metric, self._mode, config, cost_attr=self.cost_attr,- seed=self._ls.seed)+ {},+ self._ls.metric,+ self._mode,+ config,+ cost_attr=self.cost_attr,+ seed=self._ls.seed,+ )if self._gs is not None:self._gs = GlobalSearch(- space=config, metric=self._metric, mode=self._mode,- sampler=self._gs._sampler)+ space=config,+ metric=self._metric,+ mode=self._mode,+ sampler=self._gs._sampler,+ )self._gs.space = configself._init_search()class CFO(BlendSearchTuner):- ''' class for CFO algorithm- '''+ """class for CFO algorithm"""- __name__ = 'CFO'+ __name__ = "CFO"def suggest(self, trial_id: str) -> Optional[Dict]:# Number of threads is 1 or 2. Thread 0 is a vacuous thread@@ -843,8 +989,7 @@ class CFO(BlendSearchTuner):return key, keydef _create_condition(self, result: Dict) -> bool:- ''' create thread condition- '''+ """create thread condition"""if self._points_to_evaluate:# still evaluating user-specified init points# we evaluate all candidate start points before we@@ -856,16 +1001,18 @@ class CFO(BlendSearchTuner):# result needs to match or exceed the best candidate start pointobj_best = min(self._ls.metric_op * r[self._ls.metric]- for r in self._candidate_start_points.values() if r)+ for r in self._candidate_start_points.values()+ if r+ )return result[self._ls.metric] * self._ls.metric_op <= obj_bestelse:return True- def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None,- error: bool = False):+ def on_trial_complete(+ self, trial_id: str, result: Optional[Dict] = None, error: bool = False+ ):super().on_trial_complete(trial_id, result, error)- if self._candidate_start_points \- and trial_id in self._candidate_start_points:+ if self._candidate_start_points and trial_id in self._candidate_start_points:# the trial is a candidate start pointself._candidate_start_points[trial_id] = resultif len(self._search_thread_pool) < 2 and not self._points_to_evaluate:diff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 67a32bb08..1d2db41ec 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -83,8 +83,6 @@ def report(_metric=None, **kwargs):return tune.report(_metric, **kwargs)else:result = kwargs- if _verbose == 2:- logger.info(f"result: {kwargs}")if _metric:result[DEFAULT_METRIC] = _metrictrial = _runner.running_trial@@ -114,7 +112,7 @@ def run(cat_hp_cost: Optional[dict] = None,metric: Optional[str] = None,mode: Optional[str] = None,- time_budget_s: Union[int, float, datetime.timedelta] = None,+ time_budget_s: Union[int, float] = None,points_to_evaluate: Optional[List[dict]] = None,evaluated_rewards: Optional[List] = None,prune_attr: Optional[str] = None,@@ -184,7 +182,7 @@ def run(metric: A string of the metric name to optimize for.mode: A string in ['min', 'max'] to specify the objective asminimization or maximization.- time_budget_s: A float of the time budget in seconds.+ time_budget_s: int or float | The time budget in seconds.points_to_evaluate: A list of initial hyperparameterconfigurations to run first.evaluated_rewards (list): If you have previously evaluated the@@ -291,6 +289,8 @@ def run(evaluated_rewards=evaluated_rewards,low_cost_partial_config=low_cost_partial_config,cat_hp_cost=cat_hp_cost,+ time_budget_s=time_budget_s,+ num_samples=num_samples,prune_attr=prune_attr,min_resource=min_resource,max_resource=max_resource,@@ -303,10 +303,12 @@ def run(if metric is None or mode is None:metric = metric or search_alg.metricmode = mode or search_alg.mode- if time_budget_s:- search_alg.set_search_properties(- None, None, config={"time_budget_s": time_budget_s}- )+ if time_budget_s or num_samples > 0:+ search_alg.set_search_properties(+ None,+ None,+ config={"time_budget_s": time_budget_s, "num_samples": num_samples},+ )scheduler = Noneif report_intermediate_result:params = {}diff --git a/test/test_restore.py b/test/test_restore.pyindex d9196d211..2315f3334 100644--- a/test/test_restore.py+++ b/test/test_restore.py@@ -6,7 +6,6 @@ import numpy as npfrom flaml.searcher.suggestion import ConcurrencyLimiterfrom flaml import tunefrom flaml import CFO-from flaml import BlendSearchclass AbstractWarmStartTest:@@ -27,28 +26,24 @@ class AbstractWarmStartTest:search_alg, cost = self.set_basic_conf()search_alg = ConcurrencyLimiter(search_alg, 1)results_exp_1 = tune.run(- cost,- num_samples=5,- search_alg=search_alg,- verbose=0,- local_dir=self.tmpdir)+ cost, num_samples=5, search_alg=search_alg, verbose=0, local_dir=self.tmpdir+ )checkpoint_path = os.path.join(self.tmpdir, self.experiment_name)search_alg.save(checkpoint_path)return results_exp_1, np.random.get_state(), checkpoint_pathdef run_explicit_restore(self, random_state, checkpoint_path):- np.random.set_state(random_state)search_alg2, cost = self.set_basic_conf()search_alg2 = ConcurrencyLimiter(search_alg2, 1)search_alg2.restore(checkpoint_path)+ np.random.set_state(random_state)return tune.run(cost, num_samples=5, search_alg=search_alg2, verbose=0)def run_full(self):np.random.seed(162)search_alg3, cost = self.set_basic_conf()search_alg3 = ConcurrencyLimiter(search_alg3, 1)- return tune.run(- cost, num_samples=10, search_alg=search_alg3, verbose=0)+ return tune.run(cost, num_samples=10, search_alg=search_alg3, verbose=0)def testReproduce(self):results_exp_1, _, _ = self.run_part_from_scratch()@@ -75,7 +70,7 @@ class CFOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):}def cost(param):- tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))+ tune.report(loss=(param["height"] - 14) ** 2 - abs(param["width"] - 3))search_alg = CFO(space=space,@@ -86,6 +81,7 @@ class CFOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):return search_alg, cost+# # # Not doing test for BS because of problems with random seed in OptunaSearch# class BlendsearchWarmStartTest(AbstractWarmStartTest, unittest.TestCase):# def set_basic_conf(self):