Skip to content

A programming framework for agentic AI

Test restore (#103)

Qingyun Wue031c2eb7d82
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py
index 245c2cc55..1fba8f04b 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/searcher/blendsearch.py
@@ -13,7 +13,8 @@ try:
from ray.tune.suggest.optuna import OptunaSearch as GlobalSearch
from ray.tune.suggest.variant_generator import generate_variants
except ImportError:
- from .suggestion import Searcher, OptunaSearch as GlobalSearch
+ from .suggestion import Searcher
+ from .suggestion import OptunaSearch as GlobalSearch
from .variant_generator import generate_variants
from .search_thread import SearchThread
from .flow2 import FLOW2 as LocalSearch
@@ -123,7 +124,11 @@ class BlendSearch(Searcher):
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, '__name__', None) != 'CFO':
- self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
+ try:
+ gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)
+ self._gs = GlobalSearch(space=space, metric=metric, mode=mode, seed=gs_seed)
+ except TypeError:
+ self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
self._ls = LocalSearch(
@@ -135,12 +140,7 @@ class BlendSearch(Searcher):
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None) -> bool:
- if self._ls.space:
- if 'time_budget_s' in config:
- self._deadline = config.get('time_budget_s') + time.time()
- if 'metric_target' in config:
- self._metric_target = config.get('metric_target')
- else:
+ if not self._ls.space:
if metric:
self._metric = metric
if self._metric_constraints:
@@ -154,6 +154,13 @@ class BlendSearch(Searcher):
if self._gs is not None:
self._gs.set_search_properties(metric, mode, config)
self._init_search()
+ if 'time_budget_s' in config:
+ time_budget_s = config['time_budget_s']
+ if time_budget_s is not None:
+ self._deadline = time_budget_s + time.time()
+ SearchThread.set_eps(time_budget_s)
+ if 'metric_target' in config:
+ self._metric_target = config.get('metric_target')
return True
def _init_search(self):
diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.py
index 22fde2fe0..599e77933 100644
--- a/flaml/searcher/search_thread.py
+++ b/flaml/searcher/search_thread.py
@@ -20,7 +20,7 @@ class SearchThread:
'''
cost_attr = 'time_total_s'
- eps = 1e-10
+ _eps = 1.0
def __init__(self, mode: str = "min",
search_alg: Optional[Searcher] = None):
@@ -40,6 +40,10 @@ class SearchThread:
self.priority = self.speed = 0
self._init_config = True
+ @classmethod
+ def set_eps(cls, time_budget_s):
+ cls._eps = max(min(time_budget_s / 1000.0, 1.0), 1e-10)
+
def suggest(self, trial_id: str) -> Optional[Dict]:
''' use the suggest() of the underlying search algorithm
'''
@@ -74,7 +78,7 @@ class SearchThread:
# calculate speed; use 0 for invalid speed temporarily
if self.obj_best2 > self.obj_best1:
self.speed = (self.obj_best2 - self.obj_best1) / (
- self.cost_total - self.cost_best2 + self.eps)
+ max(self.cost_total - self.cost_best2, SearchThread._eps))
else:
self.speed = 0
diff --git a/flaml/searcher/suggestion.py b/flaml/searcher/suggestion.py
index 0bc064e67..7809da833 100644
--- a/flaml/searcher/suggestion.py
+++ b/flaml/searcher/suggestion.py
@@ -470,6 +470,7 @@ class OptunaSearch(Searcher):
configurations.
sampler (optuna.samplers.BaseSampler): Optuna sampler used to
draw hyperparameter configurations. Defaults to ``TPESampler``.
+ seed (int): The random seed for the sampler
Tune automatically converts search spaces to Optuna's format:
.. code-block:: python
from ray.tune.suggest.optuna import OptunaSearch
@@ -502,7 +503,8 @@ class OptunaSearch(Searcher):
metric: Optional[str] = None,
mode: Optional[str] = None,
points_to_evaluate: Optional[List[Dict]] = None,
- sampler: Optional[BaseSampler] = None):
+ sampler: Optional[BaseSampler] = None,
+ seed: Optional[int] = None):
assert ot is not None, (
"Optuna must be installed! Run `pip install optuna`.")
super(OptunaSearch, self).__init__(
@@ -524,7 +526,7 @@ class OptunaSearch(Searcher):
self._points_to_evaluate = points_to_evaluate
self._study_name = "optuna" # Fixed study name for in-memory storage
- self._sampler = sampler or ot.samplers.TPESampler()
+ self._sampler = sampler or ot.samplers.TPESampler(seed=seed)
assert isinstance(self._sampler, BaseSampler), \
"You can only pass an instance of `optuna.samplers.BaseSampler` " \
"as a sampler to `OptunaSearcher`."
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 4fdb45112..4a3b02cf8 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -4,6 +4,7 @@
* project root for license information.
'''
from typing import Optional, Union, List, Callable, Tuple
+import numpy as np
import datetime
import time
try:
@@ -315,6 +316,8 @@ def run(training_function,
mode=mode,
)
num_trials = 0
+ if time_budget_s is None:
+ time_budget_s = np.inf
while time.time() - time_start < time_budget_s and (
num_samples < 0 or num_trials < num_samples):
trial_to_run = _runner.step()
diff --git a/flaml/version.py b/flaml/version.py
index dd9b22ccc..722515271 100644
--- a/flaml/version.py
+++ b/flaml/version.py
@@ -1 +1 @@
-__version__ = "0.5.1"
+__version__ = "0.5.2"
diff --git a/test/test_restore.py b/test/test_restore.py
new file mode 100644
index 000000000..d9196d211
--- /dev/null
+++ b/test/test_restore.py
@@ -0,0 +1,107 @@
+import os
+import shutil
+import tempfile
+import unittest
+import numpy as np
+from flaml.searcher.suggestion import ConcurrencyLimiter
+from flaml import tune
+from flaml import CFO
+from flaml import BlendSearch
+
+
+class AbstractWarmStartTest:
+ def setUp(self):
+ # ray.init(num_cpus=1, local_mode=True)
+ self.tmpdir = tempfile.mkdtemp()
+ self.experiment_name = "searcher-state-Test.pkl"
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+ # ray.shutdown()
+
+ def set_basic_conf(self):
+ raise NotImplementedError()
+
+ def run_part_from_scratch(self):
+ np.random.seed(162)
+ search_alg, cost = self.set_basic_conf()
+ search_alg = ConcurrencyLimiter(search_alg, 1)
+ results_exp_1 = tune.run(
+ cost,
+ num_samples=5,
+ search_alg=search_alg,
+ verbose=0,
+ local_dir=self.tmpdir)
+ checkpoint_path = os.path.join(self.tmpdir, self.experiment_name)
+ search_alg.save(checkpoint_path)
+ return results_exp_1, np.random.get_state(), checkpoint_path
+
+ def run_explicit_restore(self, random_state, checkpoint_path):
+ np.random.set_state(random_state)
+ search_alg2, cost = self.set_basic_conf()
+ search_alg2 = ConcurrencyLimiter(search_alg2, 1)
+ search_alg2.restore(checkpoint_path)
+ return tune.run(cost, num_samples=5, search_alg=search_alg2, verbose=0)
+
+ def run_full(self):
+ np.random.seed(162)
+ search_alg3, cost = self.set_basic_conf()
+ search_alg3 = ConcurrencyLimiter(search_alg3, 1)
+ return tune.run(
+ cost, num_samples=10, search_alg=search_alg3, verbose=0)
+
+ def testReproduce(self):
+ results_exp_1, _, _ = self.run_part_from_scratch()
+ results_exp_2, _, _ = self.run_part_from_scratch()
+ trials_1_config = [trial.config for trial in results_exp_1.trials]
+ trials_2_config = [trial.config for trial in results_exp_2.trials]
+ self.assertEqual(trials_1_config, trials_2_config)
+
+ def testWarmStart(self):
+ results_exp_1, r_state, checkpoint_path = self.run_part_from_scratch()
+ results_exp_2 = self.run_explicit_restore(r_state, checkpoint_path)
+ results_exp_3 = self.run_full()
+ trials_1_config = [trial.config for trial in results_exp_1.trials]
+ trials_2_config = [trial.config for trial in results_exp_2.trials]
+ trials_3_config = [trial.config for trial in results_exp_3.trials]
+ self.assertEqual(trials_1_config + trials_2_config, trials_3_config)
+
+
+class CFOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
+ def set_basic_conf(self):
+ space = {
+ "height": tune.uniform(-100, 100),
+ "width": tune.randint(0, 100),
+ }
+
+ def cost(param):
+ tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))
+
+ search_alg = CFO(
+ space=space,
+ metric="loss",
+ mode="min",
+ seed=20,
+ )
+
+ return search_alg, cost
+
+# # # Not doing test for BS because of problems with random seed in OptunaSearch
+# class BlendsearchWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
+# def set_basic_conf(self):
+# space = {
+# "height": tune.uniform(-100, 100),
+# "width": tune.randint(0, 100),
+# }
+
+# def cost(param):
+# tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))
+
+# search_alg = BlendSearch(
+# space=space,
+# metric="loss",
+# mode="min",
+# seed=20,
+# )
+
+# return search_alg, cost