A programming framework for agentic AI
add cost budget; move loc of make_dir (#888)
Chi Wang3b6bfc2876e6
diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.pyindex d091df9c7..e7aa2e4b6 100644--- a/flaml/tune/searcher/blendsearch.py+++ b/flaml/tune/searcher/blendsearch.py@@ -62,6 +62,7 @@ class BlendSearch(Searcher):metric_constraints: Optional[List[Tuple[str, str, float]]] = None,seed: Optional[int] = 20,cost_attr: Optional[str] = "auto",+ cost_budget: Optional[float] = None,experimental: Optional[bool] = False,lexico_objectives: Optional[dict] = None,use_incumbent_result_in_evaluation=False,@@ -111,10 +112,12 @@ class BlendSearch(Searcher):metric_constraints: A list of metric constraints to be satisfied.E.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".seed: An integer of the random seed.- cost_attr: Choose from ["auto", None] to specify the attribute to evaluate the cost of different trials.- Default is "auto", which means that we will automatically chose the cost attribute to use (depending+ cost_attr: None or str to specify the attribute to evaluate the cost of different trials.+ Default is "auto", which means that we will automatically choose the cost attribute to use (dependingon the nature of the resource budget). When cost_attr is set to None, cost differences between different trials will be omitted- in our search algorithm.+ in our search algorithm. When cost_attr is set to a str different from "auto" and "time_total_s",+ this cost_attr must be available in the result dict of the trial.+ cost_budget: A float of the cost budget. Only valid when cost_attr is a str different from "auto" and "time_total_s".lexico_objectives: dict, default=None | It specifics information needed to perform multi-objectiveoptimization with lexicographic preferences. This is only supported in CFO currently.When lexico_objectives is not None, the arguments metric, mode will be invalid.@@ -154,8 +157,10 @@ class BlendSearch(Searcher):self.cost_attr = TIME_TOTAL_Selse:self.cost_attr = None+ self._cost_budget = Noneelse:self.cost_attr = cost_attr+ self._cost_budget = cost_budgetself.penalty = PENALTY # penalty term for constraintsself._metric, self._mode = metric, modeself._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation@@ -388,6 +393,7 @@ class BlendSearch(Searcher):i = 0# config_signature: tuple -> result: Dictself._result = {}+ self._cost_used = 0while self._evaluated_rewards:# go over the evaluated rewardstrial_id = f"trial_for_evaluated_{i}"@@ -467,6 +473,7 @@ class BlendSearch(Searcher):if error: # remove from result cachedel self._result[signature]else: # add to result cache+ self._cost_used += result.get(self.cost_attr, 0)self._result[signature] = result# update target metric if improvedobjective = result[self._ls.metric]@@ -702,9 +709,9 @@ class BlendSearch(Searcher):def suggest(self, trial_id: str) -> Optional[Dict]:"""choose thread, suggest a valid config."""if self._init_used and not self._points_to_evaluate:+ if self._cost_budget and self._cost_used >= self._cost_budget:+ returnchoice, backup = self._select_thread()- # if choice < 0: # timeout- # return Noneconfig = self._search_thread_pool[choice].suggest(trial_id)if not choice and config is not None and self._ls.resource:config[self._ls.resource_attr] = self.best_resource@@ -717,19 +724,19 @@ class BlendSearch(Searcher):self._search_thread_pool[choice].space,)del self._search_thread_pool[choice]- return None+ return# preliminary check; not checking config validationspace = self._search_thread_pool[choice].spaceskip = self._should_skip(choice, trial_id, config, space)use_rs = 0if skip:if choice:- return None+ return# use rs when BO fails to suggest a configconfig, space = self._ls.complete_config({})skip = self._should_skip(-1, trial_id, config, space)if skip:- return None+ returnuse_rs = 1if choice or self._valid(config,@@ -756,7 +763,7 @@ class BlendSearch(Searcher):space = thread.spaceskip = self._should_skip(backup, trial_id, config, space)if skip:- return None+ returnself._trial_proposed_by[trial_id] = backupchoice = backupif not choice: # global search@@ -801,14 +808,14 @@ class BlendSearch(Searcher):if reward is None:result = self._result.get(config_signature)if result: # tried before- return None+ returnelif result is None: # not tried beforeif self._violate_config_constriants(config, config_signature):# violate config constraints- return None+ returnself._result[config_signature] = {}else: # running but no result yet- return None+ returnself._init_used = Trueself._trial_proposed_by[trial_id] = 0self._search_thread_pool[0].running += 1@@ -817,7 +824,7 @@ class BlendSearch(Searcher):result = {self._metric: reward, self.cost_attr: 1, "config": config}# result = self._result[config_signature]self.on_trial_complete(trial_id, result)- return None+ returnif self._use_incumbent_result_in_evaluation:if self._trial_proposed_by[trial_id] > 0:choice_thread = self._search_thread_pool[@@ -900,6 +907,8 @@ class BlendSearch(Searcher):time_used = now - self._start_time + self._time_usedmin_eci = min(min_eci, time_used / num_finished * num_left)# print(f"{min_eci}, {time_used / num_finished * num_left}, {num_finished}, {num_left}")+ elif self.cost_attr is not None and self._cost_budget:+ min_eci = max(self._cost_budget - self._cost_used, 0)elif self._num_samples and self._num_samples > 0:num_finished = len(self._result)num_proposed = num_finished + len(self._trial_proposed_by)diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.pyindex f6366c9a1..9bb58a8ea 100644--- a/flaml/tune/searcher/search_thread.py+++ b/flaml/tune/searcher/search_thread.py@@ -17,6 +17,7 @@ except (ImportError, AssertionError):from .suggestion import Searcherfrom .flow2 import FLOW2from ..space import add_cost_to_space, unflatten_hierarchical+from ..result import TIME_TOTAL_Simport logginglogger = logging.getLogger(__name__)@@ -29,7 +30,7 @@ class SearchThread:self,mode: str = "min",search_alg: Optional[Searcher] = None,- cost_attr: Optional[str] = "time_total_s",+ cost_attr: Optional[str] = TIME_TOTAL_S,eps: Optional[float] = 1.0,):"""When search_alg is omitted, use local search FLOW2."""diff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 4b517c988..a6046cd83 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -447,7 +447,11 @@ def run(old_verbose = _verboseold_running_trial = _running_trialold_training_iteration = _training_iteration- if local_dir and not log_file_name and verbose > 0:+ if log_file_name:+ dir_name = os.path.dirname(log_file_name)+ if dir_name:+ os.makedirs(dir_name, exist_ok=True)+ elif local_dir and verbose > 0:os.makedirs(local_dir, exist_ok=True)log_file_name = os.path.join(local_dir, "tune_" + str(datetime.datetime.now()).replace(":", "-") + ".log"@@ -472,9 +476,6 @@ def run(logger.addHandler(old_handlers[0])if verbose > 0:if log_file_name:- dir_name = os.path.dirname(log_file_name)- if dir_name:- os.makedirs(dir_name, exist_ok=True)logger.addHandler(logging.FileHandler(log_file_name))elif not logger.hasHandlers():# Add the console handler.diff --git a/test/tune/test_searcher.py b/test/tune/test_searcher.pyindex ff29b522c..cb8c9e06a 100644--- a/test/tune/test_searcher.py+++ b/test/tune/test_searcher.py@@ -31,7 +31,7 @@ def wrong_define_search_space(trial):return {1: 1}-def test_searcher():+def test_searchers():from flaml.tune.searcher.suggestion import (OptunaSearch,Searcher,@@ -303,6 +303,13 @@ def test_searcher():from flaml import tunetune.run(lambda x: 1, config={}, use_ray=use_ray, log_file_name="logs/searcher.log")+ searcher = BlendSearch(+ space=config, cost_attr="cost", cost_budget=10, metric="m", mode="min"+ )+ analysis = tune.run(+ lambda x: {"cost": 2, "m": x["b"]}, search_alg=searcher, num_samples=10+ )+ assert len(analysis.trials) == 5def test_no_optuna():