Skip to content

A programming framework for agentic AI

Makes the evaluation_function could receive the incumbent best result as input in Tune (#339)

Z.sk7b24662dca1b
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py
index 9de56c203..39aa7f058 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/searcher/blendsearch.py
@@ -18,6 +18,7 @@ except (ImportError, AssertionError):
from .suggestion import Searcher
from .suggestion import OptunaSearch as GlobalSearch
from ..tune.trial import unflatten_dict, flatten_dict
+from ..tune import INCUMBENT_RESULT
from .search_thread import SearchThread
from .flow2 import FLOW2
from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_func
@@ -56,6 +57,7 @@ class BlendSearch(Searcher):
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
seed: Optional[int] = 20,
experimental: Optional[bool] = False,
+ use_incumbent_result_in_evaluation=False,
):
"""Constructor.
@@ -118,6 +120,7 @@ class BlendSearch(Searcher):
experimental: A bool of whether to use experimental features.
"""
self._metric, self._mode = metric, mode
+ self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
init_config = low_cost_partial_config or {}
if not init_config:
logger.info(
@@ -734,6 +737,12 @@ class BlendSearch(Searcher):
result = {self._metric: reward, self.cost_attr: 1, "config": config}
self.on_trial_complete(trial_id, result)
return None
+ if self._use_incumbent_result_in_evaluation:
+ if self._trial_proposed_by[trial_id] > 0:
+ choice_thread = self._search_thread_pool[
+ self._trial_proposed_by[trial_id]
+ ]
+ config[INCUMBENT_RESULT] = choice_thread.best_result
return config
def _should_skip(self, choice, trial_id, config, space) -> bool:
diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.py
index 2bad28da5..a148256b8 100644
--- a/flaml/searcher/search_thread.py
+++ b/flaml/searcher/search_thread.py
@@ -42,6 +42,7 @@ class SearchThread:
self.obj_best1 = self.obj_best2 = getattr(
search_alg, "best_obj", np.inf
) # inherently minimize
+ self.best_result = None
# eci: estimated cost for improvement
self.eci = self.cost_best
self.priority = self.speed = 0
@@ -139,12 +140,13 @@ class SearchThread:
self.cost_total += self.cost_last
if self._search_alg.metric in result:
obj = result[self._search_alg.metric] * self._metric_op
- if obj < self.obj_best1:
+ if obj < self.obj_best1 or self.best_result is None:
self.cost_best2 = self.cost_best1
self.cost_best1 = self.cost_total
self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
self.obj_best1 = obj
self.cost_best = self.cost_last
+ self.best_result = result
self._update_speed()
self.running -= 1
assert self.running >= 0
diff --git a/flaml/tune/__init__.py b/flaml/tune/__init__.py
index 111b1a532..755159339 100644
--- a/flaml/tune/__init__.py
+++ b/flaml/tune/__init__.py
@@ -1,12 +1,35 @@
try:
from ray import __version__ as ray_version
- assert ray_version >= '1.0.0'
- from ray.tune import (uniform, quniform, choice, randint, qrandint, randn,
- qrandn, loguniform, qloguniform, lograndint, qlograndint)
+
+ assert ray_version >= "1.0.0"
+ from ray.tune import (
+ uniform,
+ quniform,
+ choice,
+ randint,
+ qrandint,
+ randn,
+ qrandn,
+ loguniform,
+ qloguniform,
+ lograndint,
+ qlograndint,
+ )
except (ImportError, AssertionError):
- from .sample import (uniform, quniform, choice, randint, qrandint, randn,
- qrandn, loguniform, qloguniform, lograndint, qlograndint)
-from .tune import run, report
+ from .sample import (
+ uniform,
+ quniform,
+ choice,
+ randint,
+ qrandint,
+ randn,
+ qrandn,
+ loguniform,
+ qloguniform,
+ lograndint,
+ qlograndint,
+ )
+from .tune import run, report, INCUMBENT_RESULT
from .sample import polynomial_expansion_set
from .sample import PolynomialExpansionSet, Categorical, Float
from .trial import Trial
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 2c9729c1d..43ab19912 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -30,6 +30,8 @@ _verbose = 0
_running_trial = None
_training_iteration = 0
+INCUMBENT_RESULT = "__incumbent_result__"
+
class ExperimentAnalysis(EA):
"""Class for storing the experiment results."""
@@ -96,6 +98,8 @@ def report(_metric=None, **kwargs):
_running_trial = trial
result["training_iteration"] = _training_iteration
result["config"] = trial.config
+ if INCUMBENT_RESULT in result["config"]:
+ del result["config"][INCUMBENT_RESULT]
for key, value in trial.config.items():
result["config/" + key] = value
_runner.process_trial_result(_runner.running_trial, result)
@@ -134,6 +138,7 @@ def run(
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
max_failure: Optional[int] = 100,
use_ray: Optional[bool] = False,
+ use_incumbent_result_in_evaluation: Optional[bool] = None,
):
"""The trigger for HPO.
@@ -351,6 +356,7 @@ def run(
reduction_factor=flaml_scheduler_reduction_factor,
config_constraints=config_constraints,
metric_constraints=metric_constraints,
+ use_incumbent_result_in_evaluation=use_incumbent_result_in_evaluation,
)
else:
if metric is None or mode is None:
@@ -360,6 +366,18 @@ def run(
from ray.tune.suggest import ConcurrencyLimiter
else:
from flaml.searcher.suggestion import ConcurrencyLimiter
+ if (
+ search_alg.__class__.__name__
+ in [
+ "BlendSearch",
+ "CFO",
+ "CFOCat",
+ ]
+ and use_incumbent_result_in_evaluation is not None
+ ):
+ search_alg.use_incumbent_result_in_evaluation = (
+ use_incumbent_result_in_evaluation
+ )
searcher = (
search_alg.searcher
if isinstance(search_alg, ConcurrencyLimiter)
diff --git a/test/tune/test_record_incumbent.py b/test/tune/test_record_incumbent.py
new file mode 100644
index 000000000..b267ee0f9
--- /dev/null
+++ b/test/tune/test_record_incumbent.py
@@ -0,0 +1,85 @@
+import numpy as np
+from flaml import tune
+from flaml.tune import INCUMBENT_RESULT
+
+
+def rosenbrock_function(config: dict):
+ funcLoss = 50
+ for key, value in config.items():
+ if key in ["x1", "x2", "x3", "x4", "x5"]:
+ funcLoss += value ** 2 - 10 * np.cos(2 * np.pi * value)
+ if INCUMBENT_RESULT in config.keys():
+ print("----------------------------------------------")
+ print("incumbent result", config[INCUMBENT_RESULT])
+ print("----------------------------------------------")
+
+ return {"funcLoss": funcLoss}
+
+
+def test_record_incumbent(method="BlendSearch"):
+
+ if method != "CFOCat":
+ search_space = {
+ "x1": tune.randint(1, 9),
+ "x2": tune.randint(1, 9),
+ "x3": tune.randint(1, 9),
+ "x4": tune.randint(1, 9),
+ "x5": tune.randint(1, 9),
+ }
+ else:
+ search_space = {
+ "x1": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ "x2": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ "x3": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ "x4": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ "x5": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ }
+
+ max_iter = 100
+ num_samples = 128
+ time_budget_s = 1
+ n_cpu = 1
+
+ if method == "BlendSearch":
+ tune.run(
+ evaluation_function=rosenbrock_function,
+ config=search_space,
+ verbose=0,
+ metric="funcLoss",
+ mode="min",
+ max_resource=max_iter,
+ min_resource=1,
+ local_dir="logs/",
+ num_samples=num_samples * n_cpu,
+ time_budget_s=time_budget_s,
+ use_incumbent_result_in_evaluation=True,
+ )
+ return
+ elif method == "CFO":
+ from flaml import CFO
+
+ algo = CFO(
+ use_incumbent_result_in_evaluation=True,
+ )
+ elif method == "CFOCat":
+ from flaml.searcher.cfo_cat import CFOCat
+
+ algo = CFOCat(
+ use_incumbent_result_in_evaluation=True,
+ )
+ else:
+ raise NotImplementedError
+ tune.run(
+ evaluation_function=rosenbrock_function,
+ metric="funcLoss",
+ mode="min",
+ config=search_space,
+ local_dir="logs/",
+ num_samples=num_samples * n_cpu,
+ time_budget_s=time_budget_s,
+ search_alg=algo,
+ )
+
+
+if __name__ == "__main__":
+ test_record_incumbent(method="BlendSearch")