A programming framework for agentic AI
Class variables, cost_attr, and reproducibility (#587)
Qingyun Wu65fa72d58395
diff --git a/flaml/config.py b/flaml/config.pyindex fd865738c..97a990cc4 100644--- a/flaml/config.py+++ b/flaml/config.py@@ -6,8 +6,10 @@N_SPLITS = 5RANDOM_SEED = 1SPLIT_RATIO = 0.1-MEM_THRES = 4 * (1024 ** 3)+MEM_THRES = 4 * (1024**3)SMALL_LARGE_THRES = 10000000MIN_SAMPLE_TRAIN = 10000CV_HOLDOUT_THRESHOLD = 100000SAMPLE_MULTIPLY_FACTOR = 4+SEARCH_THREAD_EPS = 1.0+PENALTY = 1e10 # penalty term for constraintsdiff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 00dddec76..0da3d5f58 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -7,7 +7,6 @@ import numpy as npimport timeimport pickle-try:from ray import __version__ as ray_version@@ -22,17 +21,19 @@ from ..tune import INCUMBENT_RESULTfrom .search_thread import SearchThreadfrom .flow2 import FLOW2from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_func+from ..tune.result import TIME_TOTAL_S+import logging+SEARCH_THREAD_EPS = 1.0+PENALTY = 1e10 # penalty term for constraintslogger = logging.getLogger(__name__)class BlendSearch(Searcher):"""class for BlendSearch algorithm."""- cost_attr = "time_total_s" # cost attribute in resultlagrange = "_lagrange" # suffix for lagrange-modified metric- penalty = 1e10 # penalty term for constraintsLocalSearch = FLOW2def __init__(@@ -56,6 +57,7 @@ class BlendSearch(Searcher):] = None,metric_constraints: Optional[List[Tuple[str, str, float]]] = None,seed: Optional[int] = 20,+ cost_attr: Optional[str] = "auto",experimental: Optional[bool] = False,use_incumbent_result_in_evaluation=False,):@@ -102,8 +104,23 @@ class BlendSearch(Searcher):metric_constraints: A list of metric constraints to be satisfied.E.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".seed: An integer of the random seed.+ cost_attr: Choose from ["auto", None] to specify the attribute to evaluate the cost of different trials.+ Default is "auto", which means that we will automatically chose the cost attribute to use (depending+ on the nature of the resource budget). When cost_attr is set to None, cost differences between different trials will be omitted+ in our search algorithm.experimental: A bool of whether to use experimental features."""+ self._eps = SEARCH_THREAD_EPS+ self._input_cost_attr = cost_attr+ if cost_attr == "auto":+ if time_budget_s is not None:+ self.cost_attr = TIME_TOTAL_S+ else:+ self.cost_attr = None+ else:+ self.cost_attr = cost_attr++ self.penalty = PENALTY # penalty term for constraintsself._metric, self._mode = metric, modeself._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluationinit_config = low_cost_partial_config or {}@@ -263,6 +280,8 @@ class BlendSearch(Searcher):self._time_used += now - self._start_timeself._start_time = nowself._set_deadline()+ if self._input_cost_attr == "auto":+ self.cost_attr = TIME_TOTAL_Sif "metric_target" in setting:self._metric_target = setting.get("metric_target")if "num_samples" in setting:@@ -276,10 +295,14 @@ class BlendSearch(Searcher):def _set_deadline(self):if self._time_budget_s is not None:self._deadline = self._time_budget_s + self._start_time- SearchThread.set_eps(self._time_budget_s)+ self._set_eps()else:self._deadline = np.inf+ def _set_eps(self):+ """set eps for search threads according to time budget"""+ self._eps = max(min(self._time_budget_s / 1000.0, 1.0), 1e-9)+def _init_search(self):"""initialize the search"""self._start_time = time.time()@@ -290,7 +313,7 @@ class BlendSearch(Searcher):self._metric_target = np.inf * self._ls.metric_opself._search_thread_pool = {# id: int -> thread: SearchThread- 0: SearchThread(self._ls.mode, self._gs)+ 0: SearchThread(self._ls.mode, self._gs, self.cost_attr, self._eps)}self._thread_count = 1 # total # threads createdself._init_used = self._ls.init_config is None@@ -462,6 +485,7 @@ class BlendSearch(Searcher):space=space,),self.cost_attr,+ self._eps,)self._thread_count += 1self._update_admissible_region(diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.pyindex 585301e08..d649e7179 100644--- a/flaml/searcher/flow2.py+++ b/flaml/searcher/flow2.py@@ -88,7 +88,9 @@ class FLOW2(Searcher):self.best_config = flatten_dict(init_config)self.resource_attr = resource_attrself.min_resource = min_resource- self.resource_multiple_factor = resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR+ self.resource_multiple_factor = (+ resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR+ )self.cost_attr = cost_attrself.max_resource = max_resourceself._resource = None@@ -316,7 +318,7 @@ class FLOW2(Searcher):self.best_obj = objself.best_config, self.step = self._configs[trial_id]self.incumbent = self.normalize(self.best_config)- self.cost_incumbent = result.get(self.cost_attr)+ self.cost_incumbent = result.get(self.cost_attr, 1)if self._resource:self._resource = self.best_config[self.resource_attr]self._num_complete4incumbent = 0@@ -339,7 +341,9 @@ class FLOW2(Searcher):# proposed by current incumbent and no betterself._num_complete4incumbent += 1cost = (- result.get(self.cost_attr) if result else self._trial_cost.get(trial_id)+ result.get(self.cost_attr, 1)+ if result+ else self._trial_cost.get(trial_id))if cost:self._cost_complete4incumbent += cost@@ -369,14 +373,14 @@ class FLOW2(Searcher):if self._resource:self._resource = config[self.resource_attr]self.incumbent = self.normalize(self.best_config)- self.cost_incumbent = result.get(self.cost_attr)+ self.cost_incumbent = result.get(self.cost_attr, 1)self._cost_complete4incumbent = 0self._num_complete4incumbent = 0self._num_proposedby_incumbent = 0self._num_allowed4incumbent = 2 * self.dimself._proposed_by.clear()self._iter_best_config = self.trial_count_complete- cost = result.get(self.cost_attr)+ cost = result.get(self.cost_attr, 1)# record the cost in case it is pruned and cost info is lostself._trial_cost[trial_id] = costdiff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.pyindex fec85a7d3..4bd2d7c19 100644--- a/flaml/searcher/search_thread.py+++ b/flaml/searcher/search_thread.py@@ -22,13 +22,12 @@ logger = logging.getLogger(__name__)class SearchThread:"""Class of global or local search thread."""- _eps = 1.0-def __init__(self,mode: str = "min",search_alg: Optional[Searcher] = None,cost_attr: Optional[str] = "time_total_s",+ eps: Optional[float] = 1.0,):"""When search_alg is omitted, use local search FLOW2."""self._search_alg = search_alg@@ -38,6 +37,7 @@ class SearchThread:self.cost_best = self.cost_last = self.cost_total = self.cost_best1 = getattr(search_alg, "cost_incumbent", 0)+ self._eps = epsself.cost_best2 = 0self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf@@ -59,10 +59,6 @@ class SearchThread:# remember const configself._const = add_cost_to_space(self.space, {}, {})- @classmethod- def set_eps(cls, time_budget_s):- cls._eps = max(min(time_budget_s / 1000.0, 1.0), 1e-9)-def suggest(self, trial_id: str) -> Optional[Dict]:"""Use the suggest() of the underlying search algorithm."""if isinstance(self._search_alg, FLOW2):@@ -107,7 +103,7 @@ class SearchThread:self.speed = ((self.obj_best2 - self.obj_best1)/ self.running- / (max(self.cost_total - self.cost_best2, SearchThread._eps))+ / (max(self.cost_total - self.cost_best2, self._eps)))else:self.speed = 0@@ -164,8 +160,9 @@ class SearchThread:# rs is used in place of optuna sometimesif not str(e).endswith("has already finished and can not be updated."):raise e- if self.cost_attr in result and self.cost_last < result[self.cost_attr]:- self.cost_last = result[self.cost_attr]+ new_cost = result.get(self.cost_attr, 1)+ if self.cost_last < new_cost:+ self.cost_last = new_cost# self._update_speed()@propertydiff --git a/test/tune/test_reproducibility.py b/test/tune/test_reproducibility.pynew file mode 100644index 000000000..8a0b6230a--- /dev/null+++ b/test/tune/test_reproducibility.py@@ -0,0 +1,126 @@+from functools import partial+++def _evaluation_fn(step, width, height):+ return (0.1 + width * step / 100) ** (-1) + height * 0.1+++def _easy_objective(use_raytune, config):+ if use_raytune:+ from ray import tune+ else:+ from flaml import tune+ # Hyperparameters+ width, height = config["width"], config["height"]++ for step in range(config["steps"]):+ # Iterative training function - can be any arbitrary training procedure+ intermediate_score = _evaluation_fn(step, width, height)+ # Feed the score back back to Tune.+ try:+ tune.report(iterations=step, mean_loss=intermediate_score)+ except StopIteration:+ print("Trial stopped", step)+ return+++def test_tune(+ smoke_test=True, externally_setup_searcher=False, use_ray=False, use_raytune=False+):+ from flaml import tune+ from flaml.searcher.blendsearch import BlendSearch++ easy_objective_custom_tune = partial(_easy_objective, use_raytune)+ search_space = {+ "steps": 100,+ "width": tune.uniform(0, 20),+ "height": tune.uniform(-100, 100),+ }+ if externally_setup_searcher:++ searcher = BlendSearch(+ space=search_space,+ time_budget_s=5,+ metric="mean_loss",+ mode="min",+ )+ assert (+ searcher.cost_attr == "time_total_s"+ ), "when time_budget_s is provided, cost_attr should be time_total_s"++ searcher = BlendSearch(+ space=search_space,+ num_samples=10,+ metric="mean_loss",+ mode="min",+ )+ assert (+ searcher.cost_attr is None+ ), "when time_budget_s is not provided, cost_attr should be None."++ searcher = BlendSearch(+ space=search_space,+ num_samples=10,+ time_budget_s=5,+ metric="mean_loss",+ mode="min",+ )+ assert (+ searcher.cost_attr == "time_total_s"+ ), "As long as time_budget_s is provided and cost_attr not otherwise specified (i.e., using the default auto value), time_total_s is used as the cost_attr"++ searcher = BlendSearch(+ space=search_space,+ num_samples=10,+ time_budget_s=5,+ metric="mean_loss",+ mode="min",+ cost_attr=None,+ )+ assert (+ searcher.cost_attr is None+ ), "When the cost_attr is explicitly specified to be None, BS should use None as the cost_attr."++ searcher = BlendSearch(+ space=search_space,+ metric="mean_loss",+ mode="min",+ )+ else:+ searcher = None++ analysis = tune.run(+ easy_objective_custom_tune,+ search_alg=searcher,+ metric="mean_loss",+ mode="min",+ num_samples=10,+ time_budget_s=5,+ use_ray=use_ray,+ config=search_space,+ )++ print("Best hyperparameters found were: ", analysis.best_config)+ print("best results", analysis.best_result)+ print("best results", analysis.results)+ return analysis.best_config+++def test_reproducibility():+ best_config_1 = test_tune(smoke_test=True)+ best_config_2 = test_tune(smoke_test=True)+ print(best_config_1)+ print(best_config_2)+ assert best_config_1 == best_config_2, "flaml.tune not reproducible"++ best_config_1 = test_tune(smoke_test=True, externally_setup_searcher=True)+ best_config_2 = test_tune(smoke_test=True, externally_setup_searcher=True)+ print(best_config_1)+ print(best_config_2)+ assert (+ best_config_1 == best_config_2+ ), "flaml.tune not reproducible when the searcher is set up externally"+++if __name__ == "__main__":+ test_reproducibility()