Skip to content

A programming framework for agentic AI

Class variables, cost_attr, and reproducibility (#587)

Qingyun Wu65fa72d58395
diff --git a/flaml/config.py b/flaml/config.py
index fd865738c..97a990cc4 100644
--- a/flaml/config.py
+++ b/flaml/config.py
@@ -6,8 +6,10 @@
N_SPLITS = 5
RANDOM_SEED = 1
SPLIT_RATIO = 0.1
-MEM_THRES = 4 * (1024 ** 3)
+MEM_THRES = 4 * (1024**3)
SMALL_LARGE_THRES = 10000000
MIN_SAMPLE_TRAIN = 10000
CV_HOLDOUT_THRESHOLD = 100000
SAMPLE_MULTIPLY_FACTOR = 4
+SEARCH_THREAD_EPS = 1.0
+PENALTY = 1e10 # penalty term for constraints
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py
index 00dddec76..0da3d5f58 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/searcher/blendsearch.py
@@ -7,7 +7,6 @@ import numpy as np
import time
import pickle
-
try:
from ray import __version__ as ray_version
@@ -22,17 +21,19 @@ from ..tune import INCUMBENT_RESULT
from .search_thread import SearchThread
from .flow2 import FLOW2
from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_func
+from ..tune.result import TIME_TOTAL_S
+
import logging
+SEARCH_THREAD_EPS = 1.0
+PENALTY = 1e10 # penalty term for constraints
logger = logging.getLogger(__name__)
class BlendSearch(Searcher):
"""class for BlendSearch algorithm."""
- cost_attr = "time_total_s" # cost attribute in result
lagrange = "_lagrange" # suffix for lagrange-modified metric
- penalty = 1e10 # penalty term for constraints
LocalSearch = FLOW2
def __init__(
@@ -56,6 +57,7 @@ class BlendSearch(Searcher):
] = None,
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
seed: Optional[int] = 20,
+ cost_attr: Optional[str] = "auto",
experimental: Optional[bool] = False,
use_incumbent_result_in_evaluation=False,
):
@@ -102,8 +104,23 @@ class BlendSearch(Searcher):
metric_constraints: A list of metric constraints to be satisfied.
E.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".
seed: An integer of the random seed.
+ cost_attr: Choose from ["auto", None] to specify the attribute to evaluate the cost of different trials.
+ Default is "auto", which means that we will automatically chose the cost attribute to use (depending
+ on the nature of the resource budget). When cost_attr is set to None, cost differences between different trials will be omitted
+ in our search algorithm.
experimental: A bool of whether to use experimental features.
"""
+ self._eps = SEARCH_THREAD_EPS
+ self._input_cost_attr = cost_attr
+ if cost_attr == "auto":
+ if time_budget_s is not None:
+ self.cost_attr = TIME_TOTAL_S
+ else:
+ self.cost_attr = None
+ else:
+ self.cost_attr = cost_attr
+
+ self.penalty = PENALTY # penalty term for constraints
self._metric, self._mode = metric, mode
self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
init_config = low_cost_partial_config or {}
@@ -263,6 +280,8 @@ class BlendSearch(Searcher):
self._time_used += now - self._start_time
self._start_time = now
self._set_deadline()
+ if self._input_cost_attr == "auto":
+ self.cost_attr = TIME_TOTAL_S
if "metric_target" in setting:
self._metric_target = setting.get("metric_target")
if "num_samples" in setting:
@@ -276,10 +295,14 @@ class BlendSearch(Searcher):
def _set_deadline(self):
if self._time_budget_s is not None:
self._deadline = self._time_budget_s + self._start_time
- SearchThread.set_eps(self._time_budget_s)
+ self._set_eps()
else:
self._deadline = np.inf
+ def _set_eps(self):
+ """set eps for search threads according to time budget"""
+ self._eps = max(min(self._time_budget_s / 1000.0, 1.0), 1e-9)
+
def _init_search(self):
"""initialize the search"""
self._start_time = time.time()
@@ -290,7 +313,7 @@ class BlendSearch(Searcher):
self._metric_target = np.inf * self._ls.metric_op
self._search_thread_pool = {
# id: int -> thread: SearchThread
- 0: SearchThread(self._ls.mode, self._gs)
+ 0: SearchThread(self._ls.mode, self._gs, self.cost_attr, self._eps)
}
self._thread_count = 1 # total # threads created
self._init_used = self._ls.init_config is None
@@ -462,6 +485,7 @@ class BlendSearch(Searcher):
space=space,
),
self.cost_attr,
+ self._eps,
)
self._thread_count += 1
self._update_admissible_region(
diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.py
index 585301e08..d649e7179 100644
--- a/flaml/searcher/flow2.py
+++ b/flaml/searcher/flow2.py
@@ -88,7 +88,9 @@ class FLOW2(Searcher):
self.best_config = flatten_dict(init_config)
self.resource_attr = resource_attr
self.min_resource = min_resource
- self.resource_multiple_factor = resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR
+ self.resource_multiple_factor = (
+ resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR
+ )
self.cost_attr = cost_attr
self.max_resource = max_resource
self._resource = None
@@ -316,7 +318,7 @@ class FLOW2(Searcher):
self.best_obj = obj
self.best_config, self.step = self._configs[trial_id]
self.incumbent = self.normalize(self.best_config)
- self.cost_incumbent = result.get(self.cost_attr)
+ self.cost_incumbent = result.get(self.cost_attr, 1)
if self._resource:
self._resource = self.best_config[self.resource_attr]
self._num_complete4incumbent = 0
@@ -339,7 +341,9 @@ class FLOW2(Searcher):
# proposed by current incumbent and no better
self._num_complete4incumbent += 1
cost = (
- result.get(self.cost_attr) if result else self._trial_cost.get(trial_id)
+ result.get(self.cost_attr, 1)
+ if result
+ else self._trial_cost.get(trial_id)
)
if cost:
self._cost_complete4incumbent += cost
@@ -369,14 +373,14 @@ class FLOW2(Searcher):
if self._resource:
self._resource = config[self.resource_attr]
self.incumbent = self.normalize(self.best_config)
- self.cost_incumbent = result.get(self.cost_attr)
+ self.cost_incumbent = result.get(self.cost_attr, 1)
self._cost_complete4incumbent = 0
self._num_complete4incumbent = 0
self._num_proposedby_incumbent = 0
self._num_allowed4incumbent = 2 * self.dim
self._proposed_by.clear()
self._iter_best_config = self.trial_count_complete
- cost = result.get(self.cost_attr)
+ cost = result.get(self.cost_attr, 1)
# record the cost in case it is pruned and cost info is lost
self._trial_cost[trial_id] = cost
diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.py
index fec85a7d3..4bd2d7c19 100644
--- a/flaml/searcher/search_thread.py
+++ b/flaml/searcher/search_thread.py
@@ -22,13 +22,12 @@ logger = logging.getLogger(__name__)
class SearchThread:
"""Class of global or local search thread."""
- _eps = 1.0
-
def __init__(
self,
mode: str = "min",
search_alg: Optional[Searcher] = None,
cost_attr: Optional[str] = "time_total_s",
+ eps: Optional[float] = 1.0,
):
"""When search_alg is omitted, use local search FLOW2."""
self._search_alg = search_alg
@@ -38,6 +37,7 @@ class SearchThread:
self.cost_best = self.cost_last = self.cost_total = self.cost_best1 = getattr(
search_alg, "cost_incumbent", 0
)
+ self._eps = eps
self.cost_best2 = 0
self.obj_best1 = self.obj_best2 = getattr(
search_alg, "best_obj", np.inf
@@ -59,10 +59,6 @@ class SearchThread:
# remember const config
self._const = add_cost_to_space(self.space, {}, {})
- @classmethod
- def set_eps(cls, time_budget_s):
- cls._eps = max(min(time_budget_s / 1000.0, 1.0), 1e-9)
-
def suggest(self, trial_id: str) -> Optional[Dict]:
"""Use the suggest() of the underlying search algorithm."""
if isinstance(self._search_alg, FLOW2):
@@ -107,7 +103,7 @@ class SearchThread:
self.speed = (
(self.obj_best2 - self.obj_best1)
/ self.running
- / (max(self.cost_total - self.cost_best2, SearchThread._eps))
+ / (max(self.cost_total - self.cost_best2, self._eps))
)
else:
self.speed = 0
@@ -164,8 +160,9 @@ class SearchThread:
# rs is used in place of optuna sometimes
if not str(e).endswith("has already finished and can not be updated."):
raise e
- if self.cost_attr in result and self.cost_last < result[self.cost_attr]:
- self.cost_last = result[self.cost_attr]
+ new_cost = result.get(self.cost_attr, 1)
+ if self.cost_last < new_cost:
+ self.cost_last = new_cost
# self._update_speed()
@property
diff --git a/test/tune/test_reproducibility.py b/test/tune/test_reproducibility.py
new file mode 100644
index 000000000..8a0b6230a
--- /dev/null
+++ b/test/tune/test_reproducibility.py
@@ -0,0 +1,126 @@
+from functools import partial
+
+
+def _evaluation_fn(step, width, height):
+ return (0.1 + width * step / 100) ** (-1) + height * 0.1
+
+
+def _easy_objective(use_raytune, config):
+ if use_raytune:
+ from ray import tune
+ else:
+ from flaml import tune
+ # Hyperparameters
+ width, height = config["width"], config["height"]
+
+ for step in range(config["steps"]):
+ # Iterative training function - can be any arbitrary training procedure
+ intermediate_score = _evaluation_fn(step, width, height)
+ # Feed the score back back to Tune.
+ try:
+ tune.report(iterations=step, mean_loss=intermediate_score)
+ except StopIteration:
+ print("Trial stopped", step)
+ return
+
+
+def test_tune(
+ smoke_test=True, externally_setup_searcher=False, use_ray=False, use_raytune=False
+):
+ from flaml import tune
+ from flaml.searcher.blendsearch import BlendSearch
+
+ easy_objective_custom_tune = partial(_easy_objective, use_raytune)
+ search_space = {
+ "steps": 100,
+ "width": tune.uniform(0, 20),
+ "height": tune.uniform(-100, 100),
+ }
+ if externally_setup_searcher:
+
+ searcher = BlendSearch(
+ space=search_space,
+ time_budget_s=5,
+ metric="mean_loss",
+ mode="min",
+ )
+ assert (
+ searcher.cost_attr == "time_total_s"
+ ), "when time_budget_s is provided, cost_attr should be time_total_s"
+
+ searcher = BlendSearch(
+ space=search_space,
+ num_samples=10,
+ metric="mean_loss",
+ mode="min",
+ )
+ assert (
+ searcher.cost_attr is None
+ ), "when time_budget_s is not provided, cost_attr should be None."
+
+ searcher = BlendSearch(
+ space=search_space,
+ num_samples=10,
+ time_budget_s=5,
+ metric="mean_loss",
+ mode="min",
+ )
+ assert (
+ searcher.cost_attr == "time_total_s"
+ ), "As long as time_budget_s is provided and cost_attr not otherwise specified (i.e., using the default auto value), time_total_s is used as the cost_attr"
+
+ searcher = BlendSearch(
+ space=search_space,
+ num_samples=10,
+ time_budget_s=5,
+ metric="mean_loss",
+ mode="min",
+ cost_attr=None,
+ )
+ assert (
+ searcher.cost_attr is None
+ ), "When the cost_attr is explicitly specified to be None, BS should use None as the cost_attr."
+
+ searcher = BlendSearch(
+ space=search_space,
+ metric="mean_loss",
+ mode="min",
+ )
+ else:
+ searcher = None
+
+ analysis = tune.run(
+ easy_objective_custom_tune,
+ search_alg=searcher,
+ metric="mean_loss",
+ mode="min",
+ num_samples=10,
+ time_budget_s=5,
+ use_ray=use_ray,
+ config=search_space,
+ )
+
+ print("Best hyperparameters found were: ", analysis.best_config)
+ print("best results", analysis.best_result)
+ print("best results", analysis.results)
+ return analysis.best_config
+
+
+def test_reproducibility():
+ best_config_1 = test_tune(smoke_test=True)
+ best_config_2 = test_tune(smoke_test=True)
+ print(best_config_1)
+ print(best_config_2)
+ assert best_config_1 == best_config_2, "flaml.tune not reproducible"
+
+ best_config_1 = test_tune(smoke_test=True, externally_setup_searcher=True)
+ best_config_2 = test_tune(smoke_test=True, externally_setup_searcher=True)
+ print(best_config_1)
+ print(best_config_2)
+ assert (
+ best_config_1 == best_config_2
+ ), "flaml.tune not reproducible when the searcher is set up externally"
+
+
+if __name__ == "__main__":
+ test_reproducibility()