A programming framework for agentic AI
Test restore (#103)
Qingyun Wue031c2eb7d82
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 245c2cc55..1fba8f04b 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -13,7 +13,8 @@ try:from ray.tune.suggest.optuna import OptunaSearch as GlobalSearchfrom ray.tune.suggest.variant_generator import generate_variantsexcept ImportError:- from .suggestion import Searcher, OptunaSearch as GlobalSearch+ from .suggestion import Searcher+ from .suggestion import OptunaSearch as GlobalSearchfrom .variant_generator import generate_variantsfrom .search_thread import SearchThreadfrom .flow2 import FLOW2 as LocalSearch@@ -123,7 +124,11 @@ class BlendSearch(Searcher):if global_search_alg is not None:self._gs = global_search_algelif getattr(self, '__name__', None) != 'CFO':- self._gs = GlobalSearch(space=space, metric=metric, mode=mode)+ try:+ gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)+ self._gs = GlobalSearch(space=space, metric=metric, mode=mode, seed=gs_seed)+ except TypeError:+ self._gs = GlobalSearch(space=space, metric=metric, mode=mode)else:self._gs = Noneself._ls = LocalSearch(@@ -135,12 +140,7 @@ class BlendSearch(Searcher):metric: Optional[str] = None,mode: Optional[str] = None,config: Optional[Dict] = None) -> bool:- if self._ls.space:- if 'time_budget_s' in config:- self._deadline = config.get('time_budget_s') + time.time()- if 'metric_target' in config:- self._metric_target = config.get('metric_target')- else:+ if not self._ls.space:if metric:self._metric = metricif self._metric_constraints:@@ -154,6 +154,13 @@ class BlendSearch(Searcher):if self._gs is not None:self._gs.set_search_properties(metric, mode, config)self._init_search()+ if 'time_budget_s' in config:+ time_budget_s = config['time_budget_s']+ if time_budget_s is not None:+ self._deadline = time_budget_s + time.time()+ SearchThread.set_eps(time_budget_s)+ if 'metric_target' in config:+ self._metric_target = config.get('metric_target')return Truedef _init_search(self):diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.pyindex 22fde2fe0..599e77933 100644--- a/flaml/searcher/search_thread.py+++ b/flaml/searcher/search_thread.py@@ -20,7 +20,7 @@ class SearchThread:'''cost_attr = 'time_total_s'- eps = 1e-10+ _eps = 1.0def __init__(self, mode: str = "min",search_alg: Optional[Searcher] = None):@@ -40,6 +40,10 @@ class SearchThread:self.priority = self.speed = 0self._init_config = True+ @classmethod+ def set_eps(cls, time_budget_s):+ cls._eps = max(min(time_budget_s / 1000.0, 1.0), 1e-10)+def suggest(self, trial_id: str) -> Optional[Dict]:''' use the suggest() of the underlying search algorithm'''@@ -74,7 +78,7 @@ class SearchThread:# calculate speed; use 0 for invalid speed temporarilyif self.obj_best2 > self.obj_best1:self.speed = (self.obj_best2 - self.obj_best1) / (- self.cost_total - self.cost_best2 + self.eps)+ max(self.cost_total - self.cost_best2, SearchThread._eps))else:self.speed = 0diff --git a/flaml/searcher/suggestion.py b/flaml/searcher/suggestion.pyindex 0bc064e67..7809da833 100644--- a/flaml/searcher/suggestion.py+++ b/flaml/searcher/suggestion.py@@ -470,6 +470,7 @@ class OptunaSearch(Searcher):configurations.sampler (optuna.samplers.BaseSampler): Optuna sampler used todraw hyperparameter configurations. Defaults to ``TPESampler``.+ seed (int): The random seed for the samplerTune automatically converts search spaces to Optuna's format:.. code-block:: pythonfrom ray.tune.suggest.optuna import OptunaSearch@@ -502,7 +503,8 @@ class OptunaSearch(Searcher):metric: Optional[str] = None,mode: Optional[str] = None,points_to_evaluate: Optional[List[Dict]] = None,- sampler: Optional[BaseSampler] = None):+ sampler: Optional[BaseSampler] = None,+ seed: Optional[int] = None):assert ot is not None, ("Optuna must be installed! Run `pip install optuna`.")super(OptunaSearch, self).__init__(@@ -524,7 +526,7 @@ class OptunaSearch(Searcher):self._points_to_evaluate = points_to_evaluateself._study_name = "optuna" # Fixed study name for in-memory storage- self._sampler = sampler or ot.samplers.TPESampler()+ self._sampler = sampler or ot.samplers.TPESampler(seed=seed)assert isinstance(self._sampler, BaseSampler), \"You can only pass an instance of `optuna.samplers.BaseSampler` " \"as a sampler to `OptunaSearcher`."diff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 4fdb45112..4a3b02cf8 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -4,6 +4,7 @@* project root for license information.'''from typing import Optional, Union, List, Callable, Tuple+import numpy as npimport datetimeimport timetry:@@ -315,6 +316,8 @@ def run(training_function,mode=mode,)num_trials = 0+ if time_budget_s is None:+ time_budget_s = np.infwhile time.time() - time_start < time_budget_s and (num_samples < 0 or num_trials < num_samples):trial_to_run = _runner.step()diff --git a/flaml/version.py b/flaml/version.pyindex dd9b22ccc..722515271 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.5.1"+__version__ = "0.5.2"diff --git a/test/test_restore.py b/test/test_restore.pynew file mode 100644index 000000000..d9196d211--- /dev/null+++ b/test/test_restore.py@@ -0,0 +1,107 @@+import os+import shutil+import tempfile+import unittest+import numpy as np+from flaml.searcher.suggestion import ConcurrencyLimiter+from flaml import tune+from flaml import CFO+from flaml import BlendSearch+++class AbstractWarmStartTest:+ def setUp(self):+ # ray.init(num_cpus=1, local_mode=True)+ self.tmpdir = tempfile.mkdtemp()+ self.experiment_name = "searcher-state-Test.pkl"++ def tearDown(self):+ shutil.rmtree(self.tmpdir)+ # ray.shutdown()++ def set_basic_conf(self):+ raise NotImplementedError()++ def run_part_from_scratch(self):+ np.random.seed(162)+ search_alg, cost = self.set_basic_conf()+ search_alg = ConcurrencyLimiter(search_alg, 1)+ results_exp_1 = tune.run(+ cost,+ num_samples=5,+ search_alg=search_alg,+ verbose=0,+ local_dir=self.tmpdir)+ checkpoint_path = os.path.join(self.tmpdir, self.experiment_name)+ search_alg.save(checkpoint_path)+ return results_exp_1, np.random.get_state(), checkpoint_path++ def run_explicit_restore(self, random_state, checkpoint_path):+ np.random.set_state(random_state)+ search_alg2, cost = self.set_basic_conf()+ search_alg2 = ConcurrencyLimiter(search_alg2, 1)+ search_alg2.restore(checkpoint_path)+ return tune.run(cost, num_samples=5, search_alg=search_alg2, verbose=0)++ def run_full(self):+ np.random.seed(162)+ search_alg3, cost = self.set_basic_conf()+ search_alg3 = ConcurrencyLimiter(search_alg3, 1)+ return tune.run(+ cost, num_samples=10, search_alg=search_alg3, verbose=0)++ def testReproduce(self):+ results_exp_1, _, _ = self.run_part_from_scratch()+ results_exp_2, _, _ = self.run_part_from_scratch()+ trials_1_config = [trial.config for trial in results_exp_1.trials]+ trials_2_config = [trial.config for trial in results_exp_2.trials]+ self.assertEqual(trials_1_config, trials_2_config)++ def testWarmStart(self):+ results_exp_1, r_state, checkpoint_path = self.run_part_from_scratch()+ results_exp_2 = self.run_explicit_restore(r_state, checkpoint_path)+ results_exp_3 = self.run_full()+ trials_1_config = [trial.config for trial in results_exp_1.trials]+ trials_2_config = [trial.config for trial in results_exp_2.trials]+ trials_3_config = [trial.config for trial in results_exp_3.trials]+ self.assertEqual(trials_1_config + trials_2_config, trials_3_config)+++class CFOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):+ def set_basic_conf(self):+ space = {+ "height": tune.uniform(-100, 100),+ "width": tune.randint(0, 100),+ }++ def cost(param):+ tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))++ search_alg = CFO(+ space=space,+ metric="loss",+ mode="min",+ seed=20,+ )++ return search_alg, cost++# # # Not doing test for BS because of problems with random seed in OptunaSearch+# class BlendsearchWarmStartTest(AbstractWarmStartTest, unittest.TestCase):+# def set_basic_conf(self):+# space = {+# "height": tune.uniform(-100, 100),+# "width": tune.randint(0, 100),+# }++# def cost(param):+# tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))++# search_alg = BlendSearch(+# space=space,+# metric="loss",+# mode="min",+# seed=20,+# )++# return search_alg, cost